gitlab大版本升级，代码迁移，api实现(11-18版本)

####################通过api迁移######################### #1.配置环境变量 /etc/profile OLD_URLhttp://192.168.1.221 #老版本gitlab的访问地址 OLD_TOKEN****** #创建root令牌并给与全部权限 NEW_URLhttp://192.168.1.40 #新版本gitlab的访问地址 NEW_TOKEN****** #创建root令牌并给与全部权限 MIGRATE_DIR/root/jh_migrate #脚本存放位置 ####################开始导出######################### #2.导出用户脚本 export_all_users.sh #!/bin/bash MIGRATE_DIR$1 OLD_URL$2 OLD_TOKEN$3 page1 all_data[] while true;do ret$(curl -s -w \nCODE:%{http_code} -H PRIVATE-TOKEN:${OLD_TOKEN} ${OLD_URL}/api/v4/users?per_page100page${page}) code$(echo $ret | grep CODE: | cut -d: -f2) body$(echo $ret | grep -v CODE:) if [ $code ! 200 ];then echo 用户接口请求失败 code:${code} break fi cnt$(echo $body | jq length) if [ $cnt -eq 0 ];then echo 用户分页读取完毕 break fi all_data$(echo $all_data $body | jq -s flatten) echo 已读取用户第${page}页本页${cnt}人 page$((page1)) done echo $all_data ${MIGRATE_DIR}/users.json echo ✅ 用户导出完成总用户数$(jq length ${MIGRATE_DIR}/users.json) chmod x export_all_users.sh 执行命令 ./export_all_users.sh $MIGRATE_DIR $OLD_URL $OLD_TOKEN #3.导出群组 export_all_groups.sh #!/bin/bash MIGRATE_DIR$1 OLD_URL$2 OLD_TOKEN$3 page1 all_data[] while true;do ret$(curl -s -w \nCODE:%{http_code} -H PRIVATE-TOKEN:${OLD_TOKEN} ${OLD_URL}/api/v4/groups?per_page100page${page}) code$(echo $ret | grep CODE: | cut -d: -f2) body$(echo $ret | grep -v CODE:) if [ $code ! 200 ];then echo 群组接口请求失败 code:${code} break fi cnt$(echo $body | jq length) if [ $cnt -eq 0 ];then echo 群组分页读取完毕 break fi all_data$(echo $all_data $body | jq -s flatten) echo 已读取群组第${page}页本页${cnt}个 page$((page1)) done echo $all_data ${MIGRATE_DIR}/groups.json echo ✅ 群组导出完成总群组$(jq length ${MIGRATE_DIR}/groups.json) chmod x export_all_groups.sh 执行命令./export_all_groups.sh $MIGRATE_DIR $OLD_URL $OLD_TOKEN #4.导出项目仓库 export_all_projects.sh #!/bin/bash MIGRATE_DIR$1 OLD_URL$2 OLD_TOKEN$3 page1 all_data[] while true;do ret$(curl -s -w \nCODE:%{http_code} -H PRIVATE-TOKEN:${OLD_TOKEN} ${OLD_URL}/api/v4/projects?per_page100page${page}) code$(echo $ret | grep CODE: | cut -d: -f2) body$(echo $ret | grep -v CODE:) if [ $code ! 200 ];then echo 项目接口请求失败 code:${code} break fi cnt$(echo $body | jq length) if [ $cnt -eq 0 ];then echo 项目分页读取完毕 break fi all_data$(echo $all_data $body | jq -s flatten) echo 已读取项目第${page}页本页${cnt}个 page$((page1)) done echo $all_data ${MIGRATE_DIR}/projects.json echo ✅ 项目导出完成总项目$(jq length ${MIGRATE_DIR}/projects.json) chmod x export_all_projects.sh 执行命令./export_all_projects.sh $MIGRATE_DIR $OLD_URL $OLD_TOKEN #5.导出成员权限 export_members.sh #!/bin/bash MIGRATE_DIR$1 OLD_URL$2 OLD_TOKEN$3 # 遍历所有项目ID导出权限 jq -r .[].id ${MIGRATE_DIR}/projects.json | while read pid; do echo 导出项目 $pid 成员权限 curl -s -H PRIVATE-TOKEN: ${OLD_TOKEN} ${OLD_URL}/api/v4/projects/${pid}/members ${MIGRATE_DIR}/project_members/${pid}.json done chmod x export_members.sh 执行命令./export_members.sh $MIGRATE_DIR $OLD_URL $OLD_TOKEN ####################开始导入######################### #6.导入用户 create_jh_users.sh #!/bin/bash MIGRATE_DIR$1 NEW_URL$2 NEW_TOKEN$3 # 遍历旧用户在极狐创建同名账号 jq -c .[] ${MIGRATE_DIR}/users.json | while read user; do email$(echo $user | jq -r .email) username$(echo $user | jq -r .username) fullname$(echo $user | jq -r .name) # 跳过无邮箱系统账号 if [ -z $email ];then echo 跳过无邮箱内置账号 $username continue fi curl -s -X POST \ -H PRIVATE-TOKEN: ${NEW_TOKEN} \ -H Content-Type: application/json \ -d { \email\:\${email}\, \username\:\${username}\, \name\:\${fullname}\, \password\:\dingyong123!\, \skip_confirmation\:true } \ ${NEW_URL}/api/v4/users echo 创建用户$username | $email done chmod x create_jh_users.sh 执行命令./create_jh_users.sh $MIGRATE_DIR $NEW_URL $NEW_TOKEN #7.导入群组 create_jh_groups_all.sh #!/bin/bash MIGRATE_DIR$1 NEW_URL$2 NEW_TOKEN$3 GROUPS_JSON${MIGRATE_DIR}/groups.json # 缓存群组路径→ID避免重复请求 declare -A group_path_id # 递归获取完整父路径 get_full_path(){ local pid$1 local path while [ $pid ! null ] [ -n $pid ]; do g$(jq -r --arg p $pid .[] | select(.id ($p|tonumber)) | .path $GROUPS_JSON) if [ -z $g ] || [ $g null ]; then break fi path${g}/${path} pid$(jq -r --arg p $pid .[] | select(.id ($p|tonumber)) | .parent_id $GROUPS_JSON) done echo ${path%/} } # 循环直到所有群组创建完成处理依赖 while true; do created0 jq -c .[] $GROUPS_JSON | while read g; do gid$(echo $g | jq -r .id) gpath$(echo $g | jq -r .path) gname$(echo $g | jq -r .name) parent_id$(echo $g | jq -r .parent_id) full_path$(get_full_path $parent_id) if [ -n $full_path ]; then full_path${full_path}/${gpath} else full_path$gpath fi # 已缓存则跳过 if [ -n ${group_path_id[$full_path]} ]; then continue fi # 查询是否已存在 res$(curl -s -H PRIVATE-TOKEN:${NEW_TOKEN} ${NEW_URL}/api/v4/groups/${full_path}) exist_id$(echo $res | jq -r .id // null) if [ $exist_id ! null ]; then group_path_id[$full_path]$exist_id continue fi # 获取父群组ID parent_real_idnull if [ $parent_id ! null ]; then parent_full$(get_full_path $parent_id) if [ -n ${group_path_id[$parent_full]} ]; then parent_real_id${group_path_id[$parent_full]} else # 父群组还未创建跳过本次 continue fi fi # 创建群组 post_data{\name\:\${gname}\,\path\:\${gpath}\,\visibility\:\private\} if [ $parent_real_id ! null ]; then post_data,\parent_id\:${parent_real_id} fi curl -s -X POST \ -H PRIVATE-TOKEN: ${NEW_TOKEN} \ -H Content-Type: application/json \ -d $post_data \ ${NEW_URL}/api/v4/groups echo ✅ 创建群组$full_path # 缓存ID new_id$(curl -s -H PRIVATE-TOKEN:${NEW_TOKEN} ${NEW_URL}/api/v4/groups/${full_path} | jq -r .id) group_path_id[$full_path]$new_id created$((created1)) done if [ $created -eq 0 ]; then break fi done echo ✅ 所有层级群组创建完毕顶层所有子群组 chmod x create_jh_groups_all.sh 执行命令 ./create_jh_groups_all.sh $MIGRATE_DIR $NEW_URL $NEW_TOKEN #8.导入空仓库 create_jh_empty_project.sh #!/bin/bash MIGRATE_DIR$1 NEW_URL$2 NEW_TOKEN$3 get_namespace_id(){ local base_url$1 local tk$2 local path$3 local final_idnull local g_resp$(curl -s -H PRIVATE-TOKEN: ${tk} ${base_url}/api/v4/groups/${path}) local is_group$(echo $g_resp | jq has(id)) if [ $is_group true ];then final_id$(echo $g_resp | jq -r .id) echo $final_id return fi local u_resp$(curl -s -H PRIVATE-TOKEN: ${tk} ${base_url}/api/v4/users?username${path}) local user_cnt$(echo $u_resp | jq length) if [ $user_cnt -gt 0 ];then final_id$(echo $u_resp | jq -r .[0].namespace_id) fi echo $final_id } jq -c .[] ${MIGRATE_DIR}/projects.json | while read proj; do ns_path$(echo $proj | jq -r .namespace.path) proj_name$(echo $proj | jq -r .name) proj_path$(echo $proj | jq -r .path) ns_id$(get_namespace_id $NEW_URL $NEW_TOKEN $ns_path) if [[ $ns_id null || -z $ns_id ]];then echo ❌ 命名空间不存在跳过项目 ${ns_path}/${proj_path} continue fi # jq自动转义特殊字符兼容低版本curl不用--arg json$(jq -n \ --arg n $proj_name \ --arg p $proj_path \ --argjson ns $ns_id \ { name: $n, path: $p, namespace_id: $ns, initialize_with_readme: false, visibility: private }) curl -s -X POST \ -H PRIVATE-TOKEN: ${NEW_TOKEN} \ -H Content-Type: application/json \ -d $json \ ${NEW_URL}/api/v4/projects echo 处理项目${ns_path}/${proj_path} done chmod x create_jh_empty_project.sh 执行命令 ./create_jh_empty_project.sh $MIGRATE_DIR $NEW_URL $NEW_TOKEN #9.迁移代码 mirror_migrate_code.sh #!/bin/bash # 这里改成你自己的信息 MIGRATE_DIR/root/jh_migrate # 旧GitLab OLD_HOST192.168.1.221 OLD_USERroot OLD_TOKEN***** # 极狐GitLab NEW_HOST192.168.1.40 NEW_USERroot NEW_TOKEN***** # TMP_GIT${MIGRATE_DIR}/tmp_git mkdir -p ${TMP_GIT} # 循环遍历所有项目 jq -c .[] ${MIGRATE_DIR}/projects.json | while read proj; do ns_path$(echo $proj | jq -r .namespace.path) proj_path$(echo $proj | jq -r .path) # 正确完整带IP认证地址 OLD_AUTH_URLhttp://${OLD_USER}:${OLD_TOKEN}${OLD_HOST}/${ns_path}/${proj_path}.git NEW_AUTH_URLhttp://${NEW_USER}:${NEW_TOKEN}${NEW_HOST}/${ns_path}/${proj_path}.git SHOW_OLD_RAWhttp://${OLD_HOST}/${ns_path}/${proj_path}.git echo 开始迁移仓库${SHOW_OLD_RAW} echo DEBUG 实际克隆地址${OLD_AUTH_URL} cd ${TMP_GIT} # 镜像克隆失败跳过 if ! git clone --mirror ${OLD_AUTH_URL}; then echo ❌ 克隆旧仓库失败跳过 ${ns_path}/${proj_path} cd ${MIGRATE_DIR} continue fi cd ${proj_path}.git git lfs fetch --all git remote set-url origin ${NEW_AUTH_URL} git push --mirror origin git lfs push --all origin cd .. rm -rf ${proj_path}.git echo ✅ 仓库 ${ns_path}/${proj_path} 迁移完成 done chmod x mirror_migrate_code.sh 执行命令 ./mirror_migrate_code.sh #10.迁移权限 restore_permission.sh #!/bin/bash NEW_URL$1 NEW_TOKEN$2 MIGRATE_DIR/root/jh_migrate USER_MAP${MIGRATE_DIR}/username_uid.map OLD_ID_DICT${MIGRATE_DIR}/oldid_path_dict.json ALL_USERS_TMP${MIGRATE_DIR}/all_users.tmp.json # 前置校验 if [ ! -f $OLD_ID_DICT ];then echo 错误缺少旧项目ID映射文件 $OLD_ID_DICT exit 1 fi # 修复点内存变量存所有项目不写文件不会出现null文件 ALL_PROJECTS[] page1 per_page1000 echo 分页拉取全量项目... while true; do page_data$(curl -s -H PRIVATE-TOKEN:${NEW_TOKEN} ${NEW_URL}/api/v4/projects?per_page${per_page}page${page}) count$(echo $page_data | jq length) if [ $count -eq 0 ];then break fi # 数组合并 ALL_PROJECTS$(echo $ALL_PROJECTS $page_data | jq -s add) page$((page1)) done # 拉取用户生成用户名映射 curl -s -H PRIVATE-TOKEN:${NEW_TOKEN} ${NEW_URL}/api/v4/users?per_page1000 $ALL_USERS_TMP jq map({key:.username, value:.id}) | from_entries $ALL_USERS_TMP $USER_MAP # 遍历权限文件 ls ${MIGRATE_DIR}/project_members/*.json | while read f; do old_pid$(basename $f .json) proj_path$(jq -r --arg o $old_pid .[$o] // empty $OLD_ID_DICT) if [ -z $proj_path ]; then echo ⚠️ 旧ID $old_pid 无路径跳过 continue fi # 直接用内存变量ALL_PROJECTS匹配不再读损坏的tmp文件 jh_pid$(echo $ALL_PROJECTS | jq -r --arg p $proj_path .[] | select(.path_with_namespace $p) | .id // empty) if [ -z $jh_pid ]; then echo ❌ 未匹配仓库 $proj_path (旧ID:$old_pid) continue fi real_jh_pid$jh_pid # 遍历成员用户名匹配 jq -c .[] $f | while read member; do uname$(echo $member | jq -r .username) access_lv$(echo $member | jq -r .access_level) if [ -z $uname ]; then echo ⚠️ $proj_path 用户名为空跳过 continue fi jh_uid$(jq -r --arg u $uname .[$u] // null $USER_MAP) if [ $jh_uid null ]; then echo ⚠️ $proj_path 用户 $uname 目标库不存在跳过 continue fi res$(curl -s -X POST \ -H PRIVATE-TOKEN:${NEW_TOKEN} \ -H Content-Type:application/json \ -d {\user_id\:$jh_uid,\access_level\:$access_lv} \ ${NEW_URL}/api/v4/projects/${real_jh_pid}/members) err$(echo $res | jq -r .message // empty) if [ -n $err ]; then echo ❌ $proj_path $uname 分配失败$err else echo ✅ $proj_path $uname 权限$access_lv 同步成功 fi done done # 清理临时用户文件 rm -f $ALL_USERS_TMP $USER_MAP echo 权限同步执行完毕 chmod x restore_permission.sh 执行命令./restore_permission.sh $NEW_URL $NEW_TOKEN