- nexus3功能强大，支持docker、pypi、java、npm等等私有仓库的搭建，但是内存占用巨大，即使只跑一个仓库。
- devpi专注pypi，缺点是不支持动态缓存包（启动后会提前下载上游源的索引元数据，这个过程耗时长，cpu、内存等资源消耗都很大），除非添加了
--offline-mode
- proxpi仅支持自动缓存用户通过它拉取的包，不支持用户手动上传包，无法满足离线内网的需要。
故
- 离线内网使用devpi
- 联网局域网使用propi
1
2
|
pip install devpi-server devpi-web
devpi-server --host 0.0.0.0 --port 3141 --offline-mode
|
可以使用docker替代,下面的docker构建在codespace中完成，以下实现基于项目rexzhang/devpi-docker魔改而来
Dockerfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
FROM python:3.10-alpine
WORKDIR /app
COPY requirements.txt .
RUN \
# install depends
apk add --no-cache --virtual .build-deps build-base libffi-dev \
&& pip install --no-cache-dir -r requirements.txt \
&& apk del .build-deps \
&& find /usr/local/lib/python*/ -type f -name '*.py[cod]' -delete \
&& find /usr/local/lib/python*/ -type d -name "__pycache__" -delete \
# prepare data path
&& mkdir /data
VOLUME /data
EXPOSE 3141
COPY entrypoint.sh supervisord.conf Dockerfile .
CMD /app/entrypoint.sh
|
entrypoint.sh脚本
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
|
#!/bin/sh
# devpi-init
if [ ! -f "/data/.serverversion" ];
then
echo "Init devpi-server"
devpi-init --no-root-pypi --serverdir /data
fi
# devpi-server
echo "Start supervisor"
supervisord -c /app/supervisord.conf
# 等待日志文件生成
echo "Waiting for /tmp/devpi-server.log to be created..."
while [ ! -f /tmp/devpi-server.log ]; do
sleep 1
done
tail -f /tmp/devpi-server.log
|
requirements.txt
1
2
3
4
|
devpi-server
devpi-client
devpi-web
supervisor
|
supervisord.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
|
[unix_http_server]
file=/tmp/supervisor.sock ; the path to the socket file
[supervisord]
logfile=/tmp/supervisord.log ; main log file; default $CWD/supervisord.log
logfile_maxbytes=50MB ; max main logfile bytes b4 rotation; default 50MB
logfile_backups=10 ; # of main logfile backups; 0 means none, default 10
loglevel=info ; log level; default info; others: debug,warn,trace
pidfile=/tmp/supervisord.pid ; supervisord pidfile; default supervisord.pid
nodaemon=false ; start in foreground if true; default false
silent=false ; no logs to stdout if true; default false
minfds=1024 ; min. avail startup file descriptors; default 1024
minprocs=200 ; min. avail process descriptors;default 200
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[supervisorctl]
serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL for a unix socket
[program:devpi-server]
command=/usr/local/bin/devpi-server --host 0.0.0.0 --port 3141 --serverdir /data --offline-mode
user = root
priority = 999
startsecs = 5
redirect_stderr = True
autostart = True
stdout_logfile = /tmp/devpi-server.log
|
1
2
3
4
5
6
7
8
|
devpi use http://192.168.2.128:3141
devpi login root --password=
# 创建一个自定义索引名，默认的root/pypi不支持上传用户的包
devpi index -c root/local
# 切换到创建的自定义索引
devpi use root/local
# 上传当前目录下所有包（*.whl、*.tar.gz）到服务器
devpi upload --from-dir .
|
1
|
pip install <包名> --index-url http://192.168.2.128:3141/root/local/ --trusted-host 192.168.2.128
|
执行以下docker cli启动
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
|
docker run -itd --restart unless-stopped \
-p 3141:5000 \
# 缓存的包映射到宿主机目录
-v /tmp/proxpi:/var/cache/proxpi \
# 复用宿主机的时间和时区
-v /etc/localtime:/etc/localtime \
-v /etc/timezone:/etc/timezone \
-e PROXPI_LOGGING_LEVEL=WARNING \
# 指定包文件的缓存目录，该目录仅储存包文件，proxpi有对应的索引数据[(保存在内存内)](https://github.com/EpicWink/proxpi/issues/33#issuecomment-1692957116)记录了哪个包储存在哪个路径，不通过proxpi直接从该路径下删除包文件，proxpi的索引数据中未作相应更改，将会导致proxpi出错：它在本地缓存目录中找不到包，也不会到上游去下载，直到重启它。
-e PROXPI_CACHE_DIR=/var/cache/proxpi \
# 上游镜像
-e PROXPI_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple/ \
# 禁止下载上游索引元数据
-e PROXPI_INDEX_TTL=0 \
--name proxpi epicwink/proxpi:latest
|
1
|
pip install <包名> --index-url http://192.168.2.128:3141/index/ --trusted-host 192.168.2.128
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
|
import os
import requests
from joblib import Parallel, delayed
url = "http://12.15.9.250:8088/service/rest/internal/ui/upload/local-pypi"
# Headers from the curl command
headers = {
"Accept": "application/json, text/plain, */*",
"Accept-Language": "zh-CN,zh;q=0.9",
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"Cookie": "psession=7a2db103-73a2-405a-9e22-20a1e88fef1d; NX-ANTI-CSRF-TOKEN=0.4838417271560995; night=0; NXSESSIONID=fe82694f-3f13-40af-8f39-6c7ae4d33b99",
"NX-ANTI-CSRF-TOKEN": "0.4838417271560995",
"Origin": "http://12.15.9.250:8088",
"Pragma": "no-cache",
"Referer": "http://12.15.9.250:8088/",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
"X-Nexus-UI": "true",
}
def upload_file(file_path):
"""
上传单个文件至指定URL
:param file_path: 文件绝对路径
"""
with open(file_path, "rb") as f:
files = {
"asset0": (
os.path.basename(file_path),
f,
"application/octet-stream",
)
}
response = requests.post(url, headers=headers, files=files, verify=False)
resjson = response.json()
if isinstance(resjson, list):
print(os.path.basename(file_path), "\t", resjson)
def process_directory(directory):
"""
批量处理指定目录下的所有 .whl 和 .tar.gz 文件
:param directory: 要处理的目录路径
"""
file_paths = [os.path.join(directory, f) for f in os.listdir(directory)]
# 使用 joblib 进行并行化处理，n_jobs=20 表示使用最多20个线程
Parallel(n_jobs=20)(delayed(upload_file)(path) for path in file_paths)
process_directory("packages")
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
|
import os
import re
import subprocess
# 正则表达式匹配形如 `8-4-4-4-12.properties` 的文件名
pattern = r"^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}\.properties$"
# 存储失败的文件路径
failed_files = []
# 遍历当前目录及其子目录
for root, _, files in os.walk("."):
for file in files:
if re.match(pattern, file):
properties_file = os.path.join(root, file)
# 读取 properties 文件内容
with open(properties_file, "r", encoding="utf-8") as f:
lines = f.readlines()
# 确保文件内容足够长，避免索引超出范围
if len(lines) > 3:
name_line = lines[3].strip() # 读取第 4 行
# 提取键值对的值部分
if "=" in name_line:
blob_name = name_line.split("=", 1)[1].strip()
# 判断 blob_name 是否以 `.whl` 或 `.gz` 结尾
if blob_name.endswith((".whl", ".gz")):
base_name = os.path.basename(blob_name) # 提取文件名
name = os.path.splitext(file)[0] # 从 properties 文件名去掉后缀
bytename = name + ".bytes" # 假设同目录存在一个 .bytes 文件
# 检查 .bytes 文件是否存在
bytename_path = os.path.join(root, bytename)
if os.path.exists(bytename_path):
new_file_path = os.path.join(root, base_name)
print(f"将文件重命名: {bytename_path} -> {new_file_path}")
# 重命名文件
os.rename(bytename_path, new_file_path)
# 执行 devpi upload
try:
# 使用绝对路径上传
abs_path = os.path.abspath(new_file_path)
result = subprocess.run(
["devpi", "upload", abs_path],
check=True,
capture_output=True,
text=True,
)
# 检查输出内容判断是否成功
if "file_upload" in result.stdout:
print(f"成功上传: {abs_path}")
# 上传成功后删除文件
os.remove(abs_path)
print(f"已删除文件: {abs_path}")
else:
print(
f"上传失败: {abs_path}, 输出: {result.stdout}"
)
failed_files.append(abs_path)
except subprocess.CalledProcessError as e:
print(f"上传失败: {abs_path}, 错误: {e.stderr}")
failed_files.append(abs_path)
else:
print(f"未找到对应的 .bytes 文件: {bytename_path}")
else:
print(f"第 4 行不包含 '='，跳过文件: {properties_file}")
else:
print(f"文件内容不足 4 行，跳过文件: {properties_file}")
# 输出所有失败的文件路径
if failed_files:
print("\n以下文件上传失败，未被删除:")
for failed_file in failed_files:
print(failed_file)
|