蔚蓝触点 Azuretouch

Azuretouch
Touch your future

将 GitLab 的备份自动上传到 腾讯COS 中

之前有个服务器装了 GitLab,设置了自动备份数据。但是没有异地备份,所以导致硬盘损坏的时候,都无法恢复。
所以利用之前的腾讯对象存储 COS 的备份代码,修改了一下,可以自动备份 GitLab 的备份文件到 COS 上,并可以设置自动保留多少天的备份。

直接上代码,使用方法参照 CentOS 中备份目录到腾讯 COS 的脚本

# -*- coding=utf-8
import sys,os
reload(sys)
sys.setdefaultencoding('utf-8')
os.chdir('/www/server/panel');
sys.path.append("class/")
import db,time,re,public
from qcloud_cos import CosConfig
from qcloud_cos import CosS3Client
from qcloud_cos import CosServiceError
from qcloud_cos import CosClientError

# 腾讯云COSV5Python SDK, 目前可以支持Python2.6与Python2.7
# pip安装指南:pip install -U cos-python-sdk-v5
# cos最新可用地域,参照https://www.qcloud.com/document/product/436/6224

#执行脚本命令
def ExecShell(cmdstring, cwd=None, timeout=None, shell=True):
    import shlex
    import datetime
    import subprocess
    import time
    if shell:
        cmdstring_list = cmdstring
    else:
        cmdstring_list = shlex.split(cmdstring)
    if timeout:
        end_time = datetime.datetime.now() + datetime.timedelta(seconds=timeout)
    sub = subprocess.Popen(cmdstring_list, cwd=cwd, stdin=subprocess.PIPE,shell=shell,bufsize=4096,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
    while sub.poll() is None:
        time.sleep(0.1)
        if timeout:
            if end_time <= datetime.datetime.now():
                raise Exception("Timeout:%s"%cmdstring)
    return sub.communicate()
                                                  
class txcos_main:
    # 设置用户属性, 包括secret_id, secret_key, region
    # appid已在配置中移除,请在参数Bucket中带上appid。Bucket由bucketname-appid组成
    secret_id  = None    # 替换为用户的secret_id
    secret_key = None    # 替换为用户的secret_key
    region     = None    # 替换为用户的region
    bucket     = None
    token      = None    # 使用临时秘钥需要传入Token,默认为空,可不填
    config     = None
    client     = None
    __error_count = 0
    __error_msg = "ERROR: 无法连接到腾讯COS服务器,请检查参数设置是否正确!"
    __setupPath = '/www/server/panel/plugin/txcos'
    __bucket_path = "/"
    def __init__(self):
        #self.__conn();
        self.token = None;
    def __conn(self):
        #if self.client: return;
        #获取配置文件
        cfile = 'plugin/txcos/config.conf';
        if not os.path.exists(cfile): self.writeFile(cfile,'');
        fp = open(cfile,'r');
        if not fp:
            print 'ERROR: 请检查config.conf文件中是否有配置信息!';
        keys = fp.read().split('|');
        if len(keys) < 4:
            keys = ['','','','','/']
        if len(keys) < 5: keys.append('/');
        self.secret_id     = keys[0];
        self.secret_key    = keys[1];
        self.region        = keys[2];
        self.bucket        = keys[3];
        self.__bucket_path = self.get_path(keys[4]);
        #try:
        self.config = CosConfig(Region=self.region, Secret_id=self.secret_id, Secret_key=self.secret_key, Token=self.token)  # 获取配置对象
        self.client = CosS3Client(self.config)
        #except Exception,ex:
        #    return None;
    #同步时间
    def sync_date(self):
        ExecShell("ntpdate 0.asia.pool.ntp.org");
    #删除文件
    def delete_file(self,filename):
        self.__conn();
        try:
            response = self.client.delete_object(
                Bucket=self.bucket,
                Key=filename
            )
            #print response['ETag']
            return None
        except Exception,ex:
            return None
    #获取所有文件
    def listFile(self):
        self.__conn();
        try:
            response = self.client.list_objects(
                Bucket    = self.bucket
            )
            return response['Contents']
        except Exception,ex:
            return "";
     #获取所有文件
    def listFileWithPath(self, path):
        self.__conn();
        try:
            if path == "": return self.listFile();
            response = self.client.list_objects(
                Bucket    = self.bucket,
                Prefix    = path
            )
            return response['Contents']
        except Exception,ex:
            return "";
    #清理多余的备份文件
    def deleteBackupWithSuffix(self, suffix, count):
        self.__conn();
        data = self.listFileWithPath(self.__bucket_path);
        data = filter(lambda item: item["Key"].endswith((suffix)), data)
        data.sort(key = lambda x:x["Key"])
        data = json.dumps(data);
        list = json.loads(data);
        #print str(len(list))
        for i in range(0,len(list) - count):
            self.delete_file(list[i]["Key"])
            print "|---已清理过期备份文件:" + list[i]["Key"]
    #备份文件
    def backupFile(self, name, filepath, count):
        self.__conn();
        startTime = time.time();
        endDate = time.strftime('%Y/%m/%d %X',time.localtime())

        files = os.listdir(filepath)
        for file in files:
          filename = filepath+'/'+file
          #if self.__bucket_path != '': self.__bucket_path += name + '/';
          print self.client
          #上传文件
          file_name = self.__bucket_path + os.path.basename(filename)
          with open(filename, 'rb') as fp:
              response = self.client.put_object(
                  Bucket=self.bucket,
                  Body=fp,
                  Key=file_name,
                  StorageClass='STANDARD',
                  CacheControl='no-cache',
                  ContentDisposition="网站["+name+"]"
              )
              #print response['ETag']
          outTime = time.time() - startTime
          log = "网站["+name+"]已成功备份到腾讯COS,用时["+str(round(outTime,2))+"]秒";
          print "★["+endDate+"] " + log
          print "|---保留最新的["+count+"]份备份"
          print "|---文件名:"+os.path.basename(filename)
          #清理本地文件
          ExecShell("rm -f " + filename)
        #清理多余备份     
        self.deleteBackupWithSuffix("gitlab_backup.tar", int(count))
        return None
    #取目录路径
    def get_path(self,path):
        if path == '/': path = '';
        if path[:1] == '/': 
            path = path[1:];
            if path[-1:] != '/': path += '/';
        return path;
    #写文件内容
    def writeFile(filename,str):
        try:
            fp = open(filename, 'w+');
            fp.write(str)
            fp.close()
            return True
        except:
            return False

if __name__ == "__main__":
    import json
    data = None
    q = txcos_main();
    type = sys.argv[1];
    if type == 'file':
        data = q.backupFile(sys.argv[2],sys.argv[3],sys.argv[4]);
        exit()
    else:
        data = 'ERROR: 参数不正确!';
    print json.dumps(data)
未经允许不得转载:蔚蓝触点 Azuretouch » 将 GitLab 的备份自动上传到 腾讯COS 中

评论 抢沙发

  • 昵称 (必填)
  • 邮箱 (必填)
  • 网址