From 03d22b7683f644b986902a669a343bb8d99dafcf Mon Sep 17 00:00:00 2001 From: bigeagle Date: Sun, 20 Nov 2016 01:10:24 +0800 Subject: [PATCH] refactor(removing unneeded script files): --- scripts/anaconda.sh | 96 -------------------- scripts/aosp.sh | 21 ----- scripts/docker.sh | 59 ------------- scripts/gen_debian_exclude.py | 13 --- scripts/gen_kali_exclude.py | 13 --- scripts/gen_ubuntu_ports_exclude.py | 13 --- scripts/gitlab-ce.sh | 65 -------------- scripts/gitlab-ci-multi-runner.sh | 69 --------------- scripts/hackage.sh | 92 ------------------- scripts/helpers/apt-download | 132 ---------------------------- scripts/homebrew.sh | 17 ---- scripts/linux.sh | 12 --- scripts/lxc-images.sh | 16 ---- scripts/mongodb.sh | 88 ------------------- scripts/nodesource.sh | 18 ---- scripts/openwrt.sh | 13 --- scripts/pypi.sh | 9 -- scripts/repo-ck.sh | 17 ---- scripts/termux.sh | 16 ---- scripts/ubuntu_debmirror.sh | 27 ------ 20 files changed, 806 deletions(-) delete mode 100755 scripts/anaconda.sh delete mode 100755 scripts/aosp.sh delete mode 100755 scripts/docker.sh delete mode 100644 scripts/gen_debian_exclude.py delete mode 100644 scripts/gen_kali_exclude.py delete mode 100644 scripts/gen_ubuntu_ports_exclude.py delete mode 100755 scripts/gitlab-ce.sh delete mode 100755 scripts/gitlab-ci-multi-runner.sh delete mode 100755 scripts/hackage.sh delete mode 100644 scripts/helpers/apt-download delete mode 100755 scripts/homebrew.sh delete mode 100755 scripts/linux.sh delete mode 100755 scripts/lxc-images.sh delete mode 100755 scripts/mongodb.sh delete mode 100644 scripts/nodesource.sh delete mode 100755 scripts/openwrt.sh delete mode 100755 scripts/pypi.sh delete mode 100755 scripts/repo-ck.sh delete mode 100755 scripts/termux.sh delete mode 100644 scripts/ubuntu_debmirror.sh diff --git a/scripts/anaconda.sh b/scripts/anaconda.sh deleted file mode 100755 index bc53fad..0000000 --- a/scripts/anaconda.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash -# requires: wget, lftp, jq -# - -set -e -set -o pipefail - -CONDA_REPO_BASE=${CONDA_REPO_BASE:-"http://repo.continuum.io"} -LOCAL_DIR_BASE="${TUNASYNC_WORKING_DIR}/pkgs" -TMP_DIR=$(mktemp -d) - -CONDA_REPOS=("free" "r" "mro" "pro") -CONDA_ARCHES=("linux-64" "linux-32" "linux-armv6l" "linux-armv7l" "linux-ppc64le" "osx-64" "osx-32" "win-64" "win-32") - -function check-and-download() { - remote_file=$1 - local_file=$2 - wget -q --spider ${remote_file} - if [ $? -eq 0 ]; then - echo "downloading ${remote_file}" - wget -q -N -O ${local_file} ${remote_file} - return - fi - return 1 -} - -function cleanup () { - echo "cleaning up" - [ -d ${TMP_DIR} ] && { - [ -f ${TMP_DIR}/repodata.json ] && rm ${TMP_DIR}/repodata.json - [ -f ${TMP_DIR}/repodata.json.bz2 ] && rm ${TMP_DIR}/repodata.json.bz2 - rmdir ${TMP_DIR} - } -} - -trap cleanup EXIT - -echo ${TMP_DIR} - -for repo in ${CONDA_REPOS[@]}; do - for arch in ${CONDA_ARCHES[@]}; do - PKG_REPO_BASE="${CONDA_REPO_BASE}/pkgs/$repo/$arch" - repodata_url="${PKG_REPO_BASE}/repodata.json" - bz2_repodata_url="${PKG_REPO_BASE}/repodata.json.bz2" - LOCAL_DIR="${LOCAL_DIR_BASE}/$repo/$arch" - [ ! -d ${LOCAL_DIR} ] && mkdir -p ${LOCAL_DIR} - tmp_repodata="${TMP_DIR}/repodata.json" - tmp_bz2_repodata="${TMP_DIR}/repodata.json.bz2" - - check-and-download ${repodata_url} ${tmp_repodata} - check-and-download ${bz2_repodata_url} ${tmp_bz2_repodata} - - jq_cmd='.packages | to_entries[] | [.key, .value.size, .value.md5] | map(tostring) | join(" ")' - bzip2 -c -d ${tmp_bz2_repodata} | jq -r "${jq_cmd}" | while read line; - do - read -a tokens <<< $line - pkgfile=${tokens[0]} - pkgsize=${tokens[1]} - pkgmd5=${tokens[2]} - - pkg_url="${PKG_REPO_BASE}/${pkgfile}" - dest_file="${LOCAL_DIR}/${pkgfile}" - - declare downloaded=false - if [ -f ${dest_file} ]; then - rsize=`stat -c "%s" ${dest_file}` - if [ ${rsize} -eq ${pkgsize} ]; then - downloaded=true - echo "Skipping ${pkgfile}, size ${pkgsize}" - fi - fi - while [ $downloaded != true ]; do - echo "downloading ${pkg_url}" - wget -q -O ${dest_file} ${pkg_url} && { - # two space for md5sum check format - echo "${pkgmd5} ${dest_file}" | md5sum -c - && downloaded=true - } - done - done - - mv -f "${TMP_DIR}/repodata.json" "${LOCAL_DIR}/repodata.json" - mv -f "${TMP_DIR}/repodata.json.bz2" "${LOCAL_DIR}/repodata.json.bz2" - done -done - -function sync_installer() { - repo_url="$1" - repo_dir="$2" - - [ ! -d "$repo_dir" ] && mkdir -p "$repo_dir" - cd $repo_dir - lftp "${repo_url}/" -e "mirror --verbose -P 5; bye" -} - -sync_installer "${CONDA_REPO_BASE}/archive/" "${TUNASYNC_WORKING_DIR}/archive/" -sync_installer "${CONDA_REPO_BASE}/miniconda/" "${TUNASYNC_WORKING_DIR}/miniconda/" diff --git a/scripts/aosp.sh b/scripts/aosp.sh deleted file mode 100755 index 71ac834..0000000 --- a/scripts/aosp.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -REPO=${REPO:-"/usr/local/bin/repo"} - -function repo_init() { - mkdir -p $TUNASYNC_WORKING_DIR - cd $TUNASYNC_WORKING_DIR - $REPO init -u https://android.googlesource.com/mirror/manifest --mirror -} - -function repo_sync() { - cd $TUNASYNC_WORKING_DIR - $REPO sync -f -} - -if [ ! -d "$TUNASYNC_WORKING_DIR/git-repo.git" ]; then - echo "Initializing AOSP mirror" - repo_init -fi - -repo_sync diff --git a/scripts/docker.sh b/scripts/docker.sh deleted file mode 100755 index a800e1f..0000000 --- a/scripts/docker.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -# reqires: wget, yum-utils - -set -e -set -o pipefail - -_here=`dirname $(realpath $0)` -. ${_here}/helpers/apt-download -APT_VERSIONS=("debian-wheezy" "debian-jessie" "ubuntu-precise" "ubuntu-trusty" "ubuntu-xenial") - -BASE_PATH="${TUNASYNC_WORKING_DIR}" -APT_PATH="${BASE_PATH}/apt/repo" -YUM_PATH="${BASE_PATH}/yum/repo" - -mkdir -p ${APT_PATH} ${YUM_PATH} - -wget -q -N -O ${BASE_PATH}/yum/gpg https://yum.dockerproject.org/gpg -wget -q -N -O ${BASE_PATH}/apt/gpg https://apt.dockerproject.org/gpg - -# YUM mirror -cache_dir="/tmp/yum-docker-cache/" -cfg="/tmp/docker-yum.conf" -cat < ${cfg} -[main] -keepcache=0 - -[centos6] -name=Docker Repository -baseurl=https://yum.dockerproject.org/repo/main/centos/6 -enabled=1 -gpgcheck=0 -gpgkey=https://yum.dockerproject.org/gpg -sslverify=0 - -[centos7] -name=Docker Repository -baseurl=https://yum.dockerproject.org/repo/main/centos/7 -enabled=1 -gpgcheck=0 -gpgkey=https://yum.dockerproject.org/gpg -sslverify=0 -EOF - -[ ! -d ${YUM_PATH}/centos6 ] && mkdir -p ${YUM_PATH}/centos6 -[ ! -d ${YUM_PATH}/centos7 ] && mkdir -p ${YUM_PATH}/centos7 -reposync -c $cfg -d -p ${YUM_PATH} -e $cache_dir -createrepo --update -v -c $cache_dir -o ${YUM_PATH}/centos6 ${YUM_PATH}/centos7 -createrepo --update -v -c $cache_dir -o ${YUM_PATH}/centos6 ${YUM_PATH}/centos7 -rm $cfg - -# APT mirror -base_url="http://apt.dockerproject.org/repo" -for version in ${APT_VERSIONS[@]}; do - apt-download-binary ${base_url} "$version" "main" "amd64" "${APT_PATH}" || true - apt-download-binary ${base_url} "$version" "main" "i386" "${APT_PATH}" || true -done - -# sync_docker "http://apt.dockerproject.org/" "${TUNASYNC_WORKING_DIR}/apt" -# sync_docker "http://yum.dockerproject.org/" "${TUNASYNC_WORKING_DIR}/yum" diff --git a/scripts/gen_debian_exclude.py b/scripts/gen_debian_exclude.py deleted file mode 100644 index f48c28f..0000000 --- a/scripts/gen_debian_exclude.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python - -ARCH_EXCLUDE = ['armel', 'alpha', 'hurd-i386', 'ia64', 'kfreebsd-amd64', 'kfreebsd-i386', 'mips', 'powerpc', 'ppc64el', 's390', 's390x', 'sparc'] - -CONTENT_EXCLUDE = ['binary-{arch}', 'installer-{arch}', 'Contents-{arch}.gz', 'Contents-udeb-{arch}.gz', 'Contents-{arch}.diff', 'arch-{arch}.files', 'arch-{arch}.list.gz', '*_{arch}.deb', '*_{arch}.udeb', '*_{arch}.changes'] - -with open("debian-exclude.txt", 'wb') as f: - f.write(".~tmp~/\n") - f.write(".*\n") - for arch in ARCH_EXCLUDE: - for content in CONTENT_EXCLUDE: - f.write(content.format(arch=arch)) - f.write('\n') diff --git a/scripts/gen_kali_exclude.py b/scripts/gen_kali_exclude.py deleted file mode 100644 index de5a031..0000000 --- a/scripts/gen_kali_exclude.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python - -ARCH_EXCLUDE = ['armel', 'armhf'] - -CONTENT_EXCLUDE = ['binary-{arch}', 'installer-{arch}', 'Contents-{arch}.gz', 'Contents-udeb-{arch}.gz', 'Contents-{arch}.diff', 'arch-{arch}.files', 'arch-{arch}.list.gz', '*_{arch}.deb', '*_{arch}.udeb', '*_{arch}.changes'] - -with open("kali-exclude.txt", 'wb') as f: - f.write(".~tmp~/\n") - f.write(".*\n") - for arch in ARCH_EXCLUDE: - for content in CONTENT_EXCLUDE: - f.write(content.format(arch=arch)) - f.write('\n') diff --git a/scripts/gen_ubuntu_ports_exclude.py b/scripts/gen_ubuntu_ports_exclude.py deleted file mode 100644 index 9990ecf..0000000 --- a/scripts/gen_ubuntu_ports_exclude.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python - -ARCH_EXCLUDE = ['powerpc', 'ppc64el', 'ia64', 'sparc', 'armel'] - -CONTENT_EXCLUDE = ['binary-{arch}', 'installer-{arch}', 'Contents-{arch}.gz', 'Contents-udeb-{arch}.gz', 'Contents-{arch}.diff', 'arch-{arch}.files', 'arch-{arch}.list.gz', '*_{arch}.deb', '*_{arch}.udeb', '*_{arch}.changes'] - -with open("ubuntu-ports-exclude.txt", 'wb') as f: - f.write(".~tmp~/\n") - f.write(".*\n") - for arch in ARCH_EXCLUDE: - for content in CONTENT_EXCLUDE: - f.write(content.format(arch=arch)) - f.write('\n') diff --git a/scripts/gitlab-ce.sh b/scripts/gitlab-ce.sh deleted file mode 100755 index 0b5c42d..0000000 --- a/scripts/gitlab-ce.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash -set -e - -_here=`dirname $(realpath $0)` -. ${_here}/helpers/apt-download - -[ -z "${LOADED_APT_DOWNLOAD}" ] && (echo "failed to load apt-download"; exit 1) - -BASE_PATH="${TUNASYNC_WORKING_DIR}" - -YUM_PATH="${BASE_PATH}/yum" - -UBUNTU_VERSIONS=("trusty" "wily") -DEBIAN_VERSIONS=("wheezy" "jessie" "stretch") -UBUNTU_PATH="${BASE_PATH}/ubuntu/" -DEBIAN_PATH="${BASE_PATH}/debian/" - -mkdir -p $UBUNTU_PATH $DEBIAN_PATH $YUM_PATH - -cache_dir="/tmp/yum-gitlab-ce-cache/" -cfg="/tmp/gitlab-ce-yum.conf" -cat < ${cfg} -[main] -keepcache=0 - -[el6] -name=el6 -baseurl=https://packages.gitlab.com/gitlab/gitlab-ce/el/6/x86_64 -repo_gpgcheck=0 -gpgcheck=0 -enabled=1 -gpgkey=https://packages.gitlab.com/gpg.key -sslverify=0 - -[el7] -name=el7 -baseurl=https://packages.gitlab.com/gitlab/gitlab-ce/el/7/x86_64 -repo_gpgcheck=0 -gpgcheck=0 -enabled=1 -gpgkey=https://packages.gitlab.com/gpg.key -sslverify=0 -EOF - -reposync -c $cfg -d -p ${YUM_PATH} -e $cache_dir -createrepo --update -v -c $cache_dir -o ${YUM_PATH}/el6 ${YUM_PATH}/el6 -createrepo --update -v -c $cache_dir -o ${YUM_PATH}/el7 ${YUM_PATH}/el7 -rm $cfg - -base_url="https://packages.gitlab.com/gitlab/gitlab-ce/ubuntu" -for version in ${UBUNTU_VERSIONS[@]}; do - apt-download-binary ${base_url} "$version" "main" "amd64" "${UBUNTU_PATH}" || true - apt-download-binary ${base_url} "$version" "main" "i386" "${UBUNTU_PATH}" || true -done -echo "Ubuntu finished" - -base_url="https://packages.gitlab.com/gitlab/gitlab-ce/debian" -for version in ${DEBIAN_VERSIONS[@]}; do - apt-download-binary ${base_url} "$version" "main" "amd64" "${DEBIAN_PATH}" || true - apt-download-binary ${base_url} "$version" "main" "i386" "${DEBIAN_PATH}" || true -done -echo "Debian finished" - - -# vim: ts=4 sts=4 sw=4 diff --git a/scripts/gitlab-ci-multi-runner.sh b/scripts/gitlab-ci-multi-runner.sh deleted file mode 100755 index 5c32569..0000000 --- a/scripts/gitlab-ci-multi-runner.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -# reqires: wget, yum-utils -set -e -set -o pipefail - -_here=`dirname $(realpath $0)` -. ${_here}/helpers/apt-download - -[ -z "${LOADED_APT_DOWNLOAD}" ] && (echo "failed to load apt-download"; exit 1) - -BASE_PATH="${TUNASYNC_WORKING_DIR}" - -YUM_PATH="${BASE_PATH}/yum" - -UBUNTU_VERSIONS=("trusty" "xenial") -DEBIAN_VERSIONS=("wheezy" "jessie" "stretch") -UBUNTU_PATH="${BASE_PATH}/ubuntu/" -DEBIAN_PATH="${BASE_PATH}/debian/" - -mkdir -p $UBUNTU_PATH $DEBIAN_PATH $YUM_PATH - -cache_dir="/tmp/yum-gitlab-runner-cache/" -cfg="/tmp/gitlab-runner-yum.conf" -cat < ${cfg} -[main] -keepcache=0 - -[el6] -name=gitlab-ci-multi-runner-el6 -baseurl=https://packages.gitlab.com/runner/gitlab-ci-multi-runner/el/6/x86_64 -repo_gpgcheck=0 -gpgcheck=0 -enabled=1 -gpgkey=https://packages.gitlab.com/gpg.key -sslverify=0 - -[el7] -name=gitlab-ci-multi-runner-el7 -baseurl=https://packages.gitlab.com/runner/gitlab-ci-multi-runner/el/7/x86_64 -repo_gpgcheck=0 -gpgcheck=0 -enabled=1 -gpgkey=https://packages.gitlab.com/gpg.key -sslverify=0 -EOF - -reposync -c $cfg -d -p ${YUM_PATH} -e $cache_dir -[ ! -d ${YUM_PATH}/el6 ] && mkdir -p ${YUM_PATH}/el6 -[ ! -d ${YUM_PATH}/el7 ] && mkdir -p ${YUM_PATH}/el7 -createrepo --update -v -c $cache_dir -o ${YUM_PATH}/el6 ${YUM_PATH}/el6 -createrepo --update -v -c $cache_dir -o ${YUM_PATH}/el7 ${YUM_PATH}/el7 -rm $cfg - -base_url="https://packages.gitlab.com/runner/gitlab-ci-multi-runner/ubuntu" -for version in ${UBUNTU_VERSIONS[@]}; do - apt-download-binary ${base_url} "$version" "main" "amd64" "${UBUNTU_PATH}" || true - apt-download-binary ${base_url} "$version" "main" "i386" "${UBUNTU_PATH}" || true -done -echo "Ubuntu finished" - -base_url="https://packages.gitlab.com/runner/gitlab-ci-multi-runner/debian" -for version in ${DEBIAN_VERSIONS[@]}; do - apt-download-binary ${base_url} "$version" "main" "amd64" "${DEBIAN_PATH}" || true - apt-download-binary ${base_url} "$version" "main" "i386" "${DEBIAN_PATH}" || true -done -echo "Debian finished" - - -# vim: ts=4 sts=4 sw=4 diff --git a/scripts/hackage.sh b/scripts/hackage.sh deleted file mode 100755 index 2068274..0000000 --- a/scripts/hackage.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/bash -set -e - -function remove_broken() { - interval=$1 - interval_file="/tmp/hackage_lastcheck" - now=`date +%s` - - if [[ -f ${interval_file} ]]; then - lastcheck=`cat ${interval_file}` - between=$(echo "${now}-${lastcheck}" | bc) - [[ $between -lt $interval ]] && echo "skip checking"; return 0 - fi - echo "start checking" - - mkdir -p "${TUNASYNC_WORKING_DIR}/package" - cd "${TUNASYNC_WORKING_DIR}/package" - - ls | while read line; do - echo -n "$line\t\t" - tar -tzf $line >/dev/null || (echo "FAIL"; rm $line) && echo "OK" - done - - echo `date +%s` > $interval_file -} - -function must_download() { - src=$1 - dst=$2 - while true; do - echo "downloading: $name" - wget "$src" -O "$dst" &>/dev/null || true - tar -tzf package/$name >/dev/null || rm package/$name && break - done -} - -function hackage_mirror() { - local_pklist="/tmp/hackage_local_pklist_$$.list" - remote_pklist="/tmp/hackage_remote_pklist_$$.list" - - cd ${TUNASYNC_WORKING_DIR} - mkdir -p package - - echo "Downloading index..." - rm index.tar.gz || true - axel http://hdiff.luite.com/packages/archive/index.tar.gz -o index.tar.gz > /dev/null - - echo "building local package list" - ls package | sed "s/\.tar\.gz$//" > $local_pklist - echo "preferred-versions" >> $local_pklist # ignore preferred-versions - - echo "building remote package list" - tar ztf index.tar.gz | (cut -d/ -f 1,2 2>/dev/null) | sed 's|/|-|' > $remote_pklist - - echo "building download list" - # substract local list from remote list - comm <(sort $remote_pklist) <(sort $local_pklist) -23 | while read pk; do - # limit concurrent level - bgcount=`jobs | wc -l` - while [[ $bgcount -ge 5 ]]; do - sleep 0.5 - bgcount=`jobs | wc -l` - done - - name="$pk.tar.gz" - if [ ! -a package/$name ]; then - must_download "http://hackage.haskell.org/package/$pk/$name" "package/$name" & - else - echo "skip existed: $name" - fi - done - - # delete redundanty files - comm <(sort $remote_pklist) <(sort $local_pklist) -13 | while read pk; do - name="$pk.tar.gz" - echo "deleting ${name}" - rm "package/$name" - done - - cp index.tar.gz 00-index.tar.gz -} - -function cleanup () { - echo "cleaning up" - [[ ! -z $local_pklist ]] && (rm $local_pklist $remote_pklist ; true) -} - -trap cleanup EXIT -remove_broken 86400 -hackage_mirror - -# vim: ts=4 sts=4 sw=4 diff --git a/scripts/helpers/apt-download b/scripts/helpers/apt-download deleted file mode 100644 index f9d0f12..0000000 --- a/scripts/helpers/apt-download +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/bash -set -e -LOADED_APT_DOWNLOAD="yes" - -function check-and-download() { - remote_file=$1 - local_file=$2 - wget -q --spider ${remote_file} - if [ $? -eq 0 ]; then - echo "downloading ${remote_file}" - wget -q -N -O ${local_file} ${remote_file} - return - fi - return 0 -} - - -function apt-download-binary() { - base_url=$1 - dist=$2 - repo=$3 - arch=$4 - dest_base_dir=$5 - if [ -z $dest_base_dir ]; then - echo "Destination directory is empty, cannot continue" - return 1 - fi - - dest_dir="${dest_base_dir}/dists/${dist}" - [ ! -d "$dest_dir" ] && mkdir -p "$dest_dir" - check-and-download "${base_url}/dists/${dist}/Contents-${arch}.gz" "${dest_dir}/Contents-${arch}.gz" || true - check-and-download "${base_url}/dists/${dist}/InRelease" "${dest_dir}/InRelease" || true - check-and-download "${base_url}/dists/${dist}/Release" "${dest_dir}/Release" - check-and-download "${base_url}/dists/${dist}/Release.gpg" "${dest_dir}/Release.gpg" || true - - # Load Package Index URLs from Release file - release_file="${dest_dir}/Release" - dest_dir="${dest_base_dir}/dists/${dist}/${repo}/binary-${arch}" - [ ! -d "$dest_dir" ] && mkdir -p "$dest_dir" - - declare pkgidx_content="" - declare cnt_start=false - declare -i checksum_len - if (grep -e '^SHA256:$' ${release_file} &>/dev/null); then - checksum_cmd="sha256sum"; checksum_regex="^SHA256:$"; checksum_len=64 - elif (grep -e '^SHA1:$' ${release_file} &>/dev/null); then - checksum_cmd="sha1sum"; checksum_regex="^SHA1:$"; checksum_len=40 - elif (grep -e '^MD5Sum:$' ${release_file} &>/dev/null); then - checksum_cmd="md5sum"; checksum_regex="^MD5sum:$"; checksum_len=32 - fi - - while read line; do - if [[ ${cnt_start} = true ]]; then - read -a tokens <<< $line - checksum=${tokens[0]} - if [[ ${#checksum} != ${checksum_len} ]]; then - break - fi - filesize=${tokens[1]} - filename=${tokens[2]} - if [[ "$filename" =~ ${repo}/binary-${arch} ]]; then - # Load package list from Packages file - pkgidx_file="${dest_base_dir}/dists/${dist}/${filename}" - dest_dir=`dirname ${pkgidx_file}` - [ ! -d "$dest_dir" ] && mkdir -p "$dest_dir" - pkglist_url="${base_url}/dists/${dist}/${filename}" - check-and-download "${pkglist_url}" ${pkgidx_file} || true - echo "${checksum} ${pkgidx_file}" | ${checksum_cmd} -c - - if [ -z "${pkgidx_content}" -a -f ${pkgidx_file} ]; then - echo "getting packages index content" - case $filename in - "*.bz2") - pkgidx_content=`bunzip2 -c ${pkgidx_file}` - ;; - "*.gz") - pkgidx_content=`gunzip -c ${pkgidx_file}` - ;; - *) - pkgidx_content=`cat ${pkgidx_file}` - ;; - esac - fi - fi - else - if [[ "$line" =~ ${checksum_regex} ]]; then - cnt_start=true - fi - fi - done < ${release_file} - - if [ -z "${pkgidx_content}" ]; then - echo "index is empty, failed" - return 1 - fi - - # Set checksum method - if (echo -e "${pkgidx_content}" | grep -e '^SHA256' &>/dev/null); then - checksum_cmd="sha256sum"; checksum_regex="^SHA256" - elif (echo -e "${pkgidx_content}" | grep -e '^SHA1' &>/dev/null); then - checksum_cmd="sha1sum"; checksum_regex="^SHA1" - elif (echo -e "${pkgidx_content}" | grep -e '^MD5sum' &>/dev/null); then - checksum_cmd="md5sum"; checksum_regex="^MD5sum" - fi - - # Download packages - (echo -e "${pkgidx_content}" | grep -e '^Filename' -e '^Size' -e ${checksum_regex} | cut -d' ' -f 2) | \ - while read pkg_filename; read pkg_size; read pkg_checksum; do - dest_filename="${dest_base_dir}/${pkg_filename}" - dest_dir=`dirname ${dest_filename}` - [ ! -d "$dest_dir" ] && mkdir -p "$dest_dir" - pkg_url="${base_url}/${pkg_filename}" - declare downloaded=false - if [ -f ${dest_filename} ]; then - rsize=`stat -c "%s" ${dest_filename}` - if [ ${rsize} -eq ${pkg_size} ]; then - downloaded=true - echo "Skipping ${pkg_filename}, size ${pkg_size}" - fi - fi - while [ $downloaded != true ]; do - echo "downloading ${pkg_url}" - wget -q -O ${dest_filename} ${pkg_url} && { - echo "${pkg_checksum} ${dest_filename}" | ${checksum_cmd} -c - && downloaded=true # two space for md5sum/sha1sum/sha256sum check format - } - done - done - - echo "Mirroring ${base_url} ${dist}, ${repo}, ${arch} done!" - -} - -# vim: ts=4 sts=4 sw=4 diff --git a/scripts/homebrew.sh b/scripts/homebrew.sh deleted file mode 100755 index 9e2abfb..0000000 --- a/scripts/homebrew.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -if [ ! -d "$TUNASYNC_WORKING_DIR" ]; then - echo "Directory not exists, fail" - exit 1 -fi - -function update_homebrew_git() { - repo_dir="$1" - cd $repo_dir - echo "==== SYNC $repo_dir START ====" - /usr/bin/timeout -s INT 3600 git remote -v update - echo "==== SYNC $repo_dir DONE ====" -} - -update_homebrew_git "$TUNASYNC_WORKING_DIR/homebrew.git" -update_homebrew_git "$TUNASYNC_WORKING_DIR/homebrew-python.git" -update_homebrew_git "$TUNASYNC_WORKING_DIR/homebrew-science.git" diff --git a/scripts/linux.sh b/scripts/linux.sh deleted file mode 100755 index dca5e3c..0000000 --- a/scripts/linux.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -if [ ! -d "$TUNASYNC_WORKING_DIR" ]; then - echo "Directory not exists, fail" - exit 1 -fi - -function update_linux_git() { - cd $TUNASYNC_WORKING_DIR - /usr/bin/timeout -s INT 3600 git remote -v update -} - -update_linux_git diff --git a/scripts/lxc-images.sh b/scripts/lxc-images.sh deleted file mode 100755 index 9440a8c..0000000 --- a/scripts/lxc-images.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -function sync_lxc_images() { - repo_url="$1" - repo_dir="$2" - - [ ! -d "$repo_dir" ] && mkdir -p "$repo_dir" - cd $repo_dir - - # lftp "${repo_url}/" -e "mirror --verbose --log=${TUNASYNC_LOG_FILE} --exclude-glob='*/SRPMS/*' -P 5 --delete --only-newer; bye" - lftp "${repo_url}/" -e "mirror --verbose -P 5 --delete --only-newer; bye" -} - - -sync_lxc_images "http://images.linuxcontainers.org/images" "${TUNASYNC_WORKING_DIR}/images" -sync_lxc_images "http://images.linuxcontainers.org/meta" "${TUNASYNC_WORKING_DIR}/meta" diff --git a/scripts/mongodb.sh b/scripts/mongodb.sh deleted file mode 100755 index 444402f..0000000 --- a/scripts/mongodb.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash -set -e - -_here=`dirname $(realpath $0)` -. ${_here}/helpers/apt-download - -[ -z "${LOADED_APT_DOWNLOAD}" ] && (echo "failed to load apt-download"; exit 1) - -BASE_PATH="${TUNASYNC_WORKING_DIR}" - -YUM_PATH="${BASE_PATH}/yum" -APT_PATH="${BASE_PATH}/apt" - -UBUNTU_VERSIONS=("trusty" "precise") -DEBIAN_VERSIONS=("wheezy") -MONGO_VERSIONS=("3.2" "3.0") -STABLE_VERSION="3.2" - -UBUNTU_PATH="${APT_PATH}/ubuntu" -DEBIAN_PATH="${APT_PATH}/debian" - -mkdir -p $UBUNTU_PATH $DEBIAN_PATH $YUM_PATH - -cache_dir="/tmp/yum-mongodb-cache/" -cfg="/tmp/mongodb-yum.conf" -cat < ${cfg} -[main] -keepcache=0 - -EOF - -for mgver in ${MONGO_VERSIONS[@]}; do -cat <> ${cfg} -[el6-${mgver}] -name=el6-${mgver} -baseurl=https://repo.mongodb.org/yum/redhat/6/mongodb-org/${mgver}/x86_64/ -repo_gpgcheck=0 -gpgcheck=0 -enabled=1 -sslverify=0 - -[el7-${mgver}] -name=el7-${mgver} -baseurl=https://repo.mongodb.org/yum/redhat/7/mongodb-org/${mgver}/x86_64/ -repo_gpgcheck=0 -gpgcheck=0 -enabled=1 -sslverify=0 -EOF -done - -reposync -c $cfg -d -p ${YUM_PATH} -e $cache_dir -for mgver in ${MONGO_VERSIONS[@]}; do - createrepo --update -v -c $cache_dir -o ${YUM_PATH}/el6-$mgver/ ${YUM_PATH}/el6-$mgver/ - createrepo --update -v -c $cache_dir -o ${YUM_PATH}/el7-$mgver/ ${YUM_PATH}/el7-$mgver/ -done - -[ -e ${YUM_PATH}/el6 ] || (cd ${YUM_PATH}; ln -s el6-${STABLE_VERSION} el6) -[ -e ${YUM_PATH}/el7 ] || (cd ${YUM_PATH}; ln -s el7-${STABLE_VERSION} el7) - -rm $cfg - -base_url="http://repo.mongodb.org/apt/ubuntu" -for ubver in ${UBUNTU_VERSIONS[@]}; do - for mgver in ${MONGO_VERSIONS[@]}; do - version="$ubver/mongodb-org/$mgver" - apt-download-binary ${base_url} "$version" "multiverse" "amd64" "${UBUNTU_PATH}" || true - apt-download-binary ${base_url} "$version" "multiverse" "i386" "${UBUNTU_PATH}" || true - done - mg_basepath="${UBUNTU_PATH}/dists/$ubver/mongodb-org" - [ -e ${mg_basepath}/stable ] || (cd ${mg_basepath}; ln -s ${STABLE_VERSION} stable) -done -echo "Ubuntu finished" - -base_url="http://repo.mongodb.org/apt/debian" -for dbver in ${DEBIAN_VERSIONS[@]}; do - for mgver in ${MONGO_VERSIONS[@]}; do - version="$dbver/mongodb-org/$mgver" - apt-download-binary ${base_url} "$version" "main" "amd64" "${DEBIAN_PATH}" || true - apt-download-binary ${base_url} "$version" "main" "i386" "${DEBIAN_PATH}" || true - done - mg_basepath="${DEBIAN_PATH}/dists/$dbver/mongodb-org" - [ -e ${mg_basepath}/stable ] || (cd ${mg_basepath}; ln -s ${STABLE_VERSION} stable) -done -echo "Debian finished" - - -# vim: ts=4 sts=4 sw=4 diff --git a/scripts/nodesource.sh b/scripts/nodesource.sh deleted file mode 100644 index 8dda8e7..0000000 --- a/scripts/nodesource.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -function sync_nodesource() { - repo_url="$1" - repo_dir="$2" - - [ ! -d "$repo_dir" ] && mkdir -p "$repo_dir" - cd $repo_dir - # lftp "${repo_url}/" -e "mirror --verbose --exclude-glob='*/SRPMS/*' -P 5 --delete --only-newer; bye" - lftp "${repo_url}/" -e "mirror --verbose -P 5 --delete --only-newer; bye" -} - -sync_nodesource "https://deb.nodesource.com/node" "${TUNASYNC_WORKING_DIR}/deb" -sync_nodesource "https://deb.nodesource.com/node_0.12" "${TUNASYNC_WORKING_DIR}/deb_0.12" -sync_nodesource "https://deb.nodesource.com/node_4.x" "${TUNASYNC_WORKING_DIR}/deb_4.x" -sync_nodesource "https://rpm.nodesource.com/pub" "${TUNASYNC_WORKING_DIR}/rpm" -sync_nodesource "https://rpm.nodesource.com/pub_0.12" "${TUNASYNC_WORKING_DIR}/rpm_0.12" -sync_nodesource "https://rpm.nodesource.com/pub_4.x" "${TUNASYNC_WORKING_DIR}/rpm_4.x" diff --git a/scripts/openwrt.sh b/scripts/openwrt.sh deleted file mode 100755 index 372d696..0000000 --- a/scripts/openwrt.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -function sync_openwrt() { - repo_url="$1" - repo_dir="$2" - - [ ! -d "$repo_dir" ] && mkdir -p "$repo_dir" - cd $repo_dir - lftp "${repo_url}/" -e "mirror --verbose -P 5 --delete --only-newer; bye" -} - -sync_openwrt "http://downloads.openwrt.org/chaos_calmer/15.05/" "${TUNASYNC_WORKING_DIR}/chaos_calmer/15.05" -sync_openwrt "http://downloads.openwrt.org/snapshots/trunk/" "${TUNASYNC_WORKING_DIR}/snapshots/trunk" diff --git a/scripts/pypi.sh b/scripts/pypi.sh deleted file mode 100755 index 5882e61..0000000 --- a/scripts/pypi.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -if [ ! -d "$TUNASYNC_WORKING_DIR" ]; then - echo "Directory not exists, fail" - exit 1 -fi - -echo "Syncing to $TUNASYNC_WORKING_DIR" - -/usr/bin/timeout -s INT 3600 /home/tuna/.virtualenvs/bandersnatch/bin/bandersnatch -c /etc/bandersnatch.conf mirror || exit 1 diff --git a/scripts/repo-ck.sh b/scripts/repo-ck.sh deleted file mode 100755 index 8e4538a..0000000 --- a/scripts/repo-ck.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -function sync_repo_ck() { - repo_url="$1" - repo_dir="$2" - - [ ! -d "$repo_dir" ] && mkdir -p "$repo_dir" - cd $repo_dir - lftp "${repo_url}/" -e 'mirror -v -P 5 --delete --only-missing --only-newer --no-recursion; bye' - wget "${repo_url}/repo-ck.db" -O "repo-ck.db" - wget "${repo_url}/repo-ck.files" -O "repo-ck.files" -} - -UPSTREAM="http://repo-ck.com" - -sync_repo_ck "${UPSTREAM}/x86_64" "${TUNASYNC_WORKING_DIR}/x86_64" -sync_repo_ck "${UPSTREAM}/i686" "${TUNASYNC_WORKING_DIR}/i686" diff --git a/scripts/termux.sh b/scripts/termux.sh deleted file mode 100755 index 9dd1002..0000000 --- a/scripts/termux.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -e - -_here=`dirname $(realpath $0)` -. ${_here}/helpers/apt-download -[ -z "${LOADED_APT_DOWNLOAD}" ] && (echo "failed to load apt-download"; exit 1) - -BASE_PATH="${TUNASYNC_WORKING_DIR}" - -base_url="http://apt.termux.com" -ARCHES=("aarch64" "all" "arm" "i686") -for arch in ${ARCHES[@]}; do - echo "start syncing: ${arch}" - apt-download-binary "${base_url}" "stable" "main" "${arch}" "${BASE_PATH}" || true -done -echo "finished" diff --git a/scripts/ubuntu_debmirror.sh b/scripts/ubuntu_debmirror.sh deleted file mode 100644 index da5fc94..0000000 --- a/scripts/ubuntu_debmirror.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -SYNC_FILES="$TUNASYNC_WORKING_DIR" -# SYNC_FILES="/srv/mirror_disk/ubuntu/_working/" -#LOG_FILE="$TUNASYNC_LOG_FILE" - -# [ -f $SYNC_LOCK ] && exit 1 -# touch $SYNC_LOCK - - -echo ">> Starting sync on $(date --rfc-3339=seconds)" - -arch="i386,amd64" -sections="main,main/debian-installer,multiverse,multiverse/debian-installer,restricted,restricted/debian-installer,universe,universe/debian-installer" -dists="precise,precise-backports,precise-proposed,precise-updates,precise-security,trusty,trusty-backports,trusty-proposed,trusty-updates,trusty-security" -server="$1" -inPath="/ubuntu" -proto="rsync" -outpath="$SYNC_FILES" -rsyncOpt='-6 -aIL --partial' - -debmirror -h $server --no-check-gpg -a $arch -s $sections -d $dists -r $inPath -e $proto --rsync-options "$rsyncOpt" --verbose $outpath - -date --rfc-3339=seconds > "$SYNC_FILES/lastsync" -echo ">> Finished sync on $(date --rfc-3339=seconds)" - -# rm -f "$SYNC_LOCK" -exit 0