测试大数据容器

docker运行hadoop

sudo docker run -dit --name hadoop --network=host --privileged=true -p 8020:8020 -p 50070:50070 -p 8088:8088 -p 9000:9000 sequenceiq/hadoop-docker:2.7.0 /etc/bootstrap.sh -bash

python操作hdfs

安装依赖

pip install hdfs

python上传文件代码

from hdfs import InsecureClient

def upload_to_hadoop(local_file_path, hdfs_destination_path):
    # 创建HDFS客户端实例，这里的url格式为 http://namenode_host:port/webhdfs/v1/
    client = InsecureClient('http://192.168.61.202:50070', root='/',user='root')

    try:
        # 将本地文件上传到HDFS
        client.upload(hdfs_destination_path, local_file_path)
        print(f'Successfully uploaded {local_file_path} to {hdfs_destination_path}')
    except Exception as e:
        print(f'Error occurred while uploading: {e}')
    finally:
        # 不需要显式关闭，因为InsecureClient对象会在适当的时候自动清理资源
        print("")

# 示例调用
local_filepath = 'F:\work\docs\云计算\Docker\python上传文件到hdfs.md'
hdfs_dest_path = '/file.txt'
upload_to_hadoop(local_filepath, hdfs_dest_path)

docker运行hbase

sudo docker run -dit --name hbase-node --network=host \
  -p 16010:16010 \
  -p 9090:9090 \
  -p 60020:60020 \
  -v /path/to/hbase-data:/hbasedata \
  harisekhon/hbase:latest

# 进入容器执行命令
sudo docker exec -it hbase-node /bin/bash

python操作hbase

安装依赖

pip install -i https://pypi.tuna.tsinghua.edu.cn/simple  happybase

相关python代码

# 导入happybase库
import happybase

# 连接到HBase
connection = happybase.Connection('192.168.61.202', port=9090)

# 创建一个新的表，假设表名为'test_table'，并定义一列族'cf'
table = connection.create_table('table1', {'cf': dict()})
print("Table created.")
# 关闭连接
connection.close()