[{"_1":2,"_155":-5,"_156":-5},"loaderData",{"_3":4,"_13":14,"_21":22,"_134":135},"root",{"_5":-5,"_6":-7,"_7":8,"_9":-7,"_10":11,"_12":-7},"currentUser","primaryEmail","siteVerification","6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI","toast","enableDev",false,"incidentStatus","routes/_layout",{"_15":-5,"_16":17,"_18":17,"_19":11,"_20":11},"username","isAskGordonEnabled",true,"isUsingNewTemplate","showMinimalLanding","showHoneypotTrap","routes/_layout.r.$namespace.$reponame",{"_23":24,"_25":-5,"_26":11,"_27":11,"_28":29,"_30":31,"_32":-5,"_33":11,"_34":35,"_84":-5,"_85":86,"_125":-5,"_126":11,"_127":17,"_128":11,"_129":-5,"_130":11,"_131":-5,"_132":133},"canonicalUrl","https://hub.docker.com/r/echo21bash/sersync/tags","mcpServer","canSeeDockerfile","canSeeAutobuilds","namespace","echo21bash","reponame","sersync","cloudBuildSource","isAutobuildsAvailable","repo",{"_36":37,"_38":39,"_41":42,"_43":44,"_45":11,"_46":47,"_48":49,"_50":51,"_53":31,"_28":29,"_54":55,"_56":40,"_57":58,"_59":60,"_61":62,"_63":-5,"_64":58,"_65":66,"_67":11,"_68":29,"_69":70,"_75":11,"_76":77,"_81":29,"_82":83},"categories",[],"content_types",[40],"image","date_registered","2023-04-12T14:56:32.062118Z","description","数据同步\nhttps://github.com/Echo21bash/ops_script/tree/master/script/inotify-tools%2Brsync","is_private","last_updated","2024-03-25T07:11:23.110743Z","last_modified","2024-10-16T13:48:34.145251Z","media_types",[52],"application/vnd.docker.container.image.v1+json","name","pull_count",93,"repository_type","star_count",0,"status",1,"status_description","active","source","collaborator_count","full_description","# 数据同步方案\n\n## 同步原理\n\n> Inotify，它是在内核 2.6.13 版本中引入的一个新功能，它为用户态监视文件系统的变化提供了强大的支持，允许监控程序打开一个独立文件描述符，并针对事件集监控一个或者多个文件，例如打开、关闭、移动/重命名、删除、创建或者改变属性。\n\n**基于inotify监听文件事件类型，然后调用rsync进行远程同步到rsyncd服务端**\n\n## 脚本特点\n\n>* 支持实时同步；\n>* 支持周期性全量同步；\n>* 支持启动时首次同步；\n>* 支持历史备份数据保留指定天数；\n>* 删除多余事件减少资源使用；\n\n## 组件说明\n\n> * inotify、rsync需要部署在需要同步的服务器；\n> * rsyncd需要部署在备份服务器用于接收客户端数据；\n> * inotify.sh脚本用于监听目录变化并处理重复事件；\n> * prsync.sh脚本是对rsync命令封装用于并发传输文件；\n> * rsync.sh脚本是对rsync命令简单封装；\n> * sersync.sh脚本用于根据传入的参数调用rsync与远程服务器(依赖rsyncd服务)进行数据增量同步；\n> * tasker是一个基于Go开发的cron表达式解析工具https://github.com/adhocore/gronx\n\n## 备份服务安装\n\n### 虚拟机部署\n\n#### 安装rsync\n\n```shell\n###安装rsync\nyum install rsync -y\n\n###创建密码验证\necho 'rsync:Ki13W@yYZvbJ' >/etc/rsyncd.secret\necho 'Ki13W@yYZvbJ' >/etc/rsync.passwd\n\n###设置权限\nchmod 600 /etc/rsyncd.conf\nchmod 600 /etc/rsyncd.secret\nchmod 600 /etc/rsync.passwd\n```\n\n#### 创建配置文件\n\n```shell\n# 配置示例，两个模块分别是backup1和backup2，hosts allow字段根据实际修改\ncat > /etc/rsyncd.conf <<'EOF'\nuid = root\ngid = root\nmax connections = 200\nport = 873\nsecrets file = /etc/rsyncd.secret\nignore errors = yes\nreverse lookup = no\nlog file = /var/log/rsyncd.log\n\n[backup1]\npath = /data/backup1\ncomment = backup1\nread only = no\nlist = no\nauth users = rsync\nhosts allow = 10.255.50.63,10.255.50.64,10.255.60.2\n\n[backup2]\npath = /data/backup2\ncomment = backup2\nread only = no\nlist = no\nauth users = rsync\nhosts allow = 10.255.50.63,10.255.50.64,10.255.60.2\nEOF\n\n###创建目录\nmkdir -p /data/backup1 /data/backup2\n```\n\n#### 创建日志分割\n\n```shell\ncat > /etc/logrotate.d/rsync <<'EOF'\n/var/log/rsyncd.log{\nnotifempty\ndaily\nrotate 7\n}\nEOF\n```\n\n#### 启动rsyncd\n\n```shell\n###启动\nsystemctl start rsyncd\nsystemctl enable rsyncd\n```\n\n### 容器部署\n\n#### 启动容器\n\n```shell\ndocker run --name rsyncd --privileged -itd -p 873:873 \\\n-e RUN_MODE=rsyncd \\\n-e RSYNCD_USER=rsync \\\n-e RSYNCD_PASSWD=\"Ki13W@yYZvbJ\" \\\n-e RSYNCD_MOD_NAME=\"filebackup\" \\\n-e RSYNCD_PATH=/data \\\n-v /data:/data \\\necho21bash/sersync:1.0\n```\n\n\n\n## 备份源安装\n\n### 虚拟机部署\n\n#### 安装inotify-tools\n\n> yum安装\n\n```shell\n##Centos7\nyum install epel-release -y\nyum install inotify-tools -y\n##Openeuler\nyum install inotify-tools -y\n```\n\n>编译安装\n\n```shell\ncd /usr/local/src/\nwget https://ghproxy.com/https://github.com/inotify-tools/inotify-tools/archive/refs/tags/3.22.6.0.tar.gz\ntar zxf 3.22.6.0.tar.gz\ncd inotify-tools-3.22.6.0/\nyum install automake autoconf libtool -y\nsh autogen.sh\n./configure && make && make install\n```\n\n#### 安装rsync\n\n```shell\nyum install rsync -y\n#认证文件\necho 'Ki13W@yYZvbJ' >/etc/rsync.passwd\nchmod 600 /etc/rsync.passwd\n```\n\n#### 内核参数优化\n\n```shell\necho 'fs.inotify.max_user_watches = 999999' >> /etc/sysctl.conf\necho 'fs.inotify.max_user_instances = 1024' >> /etc/sysctl.conf\necho 'fs.inotify.max_queued_events = 999999' >> /etc/sysctl.conf\nsysctl -p\n```\n\n#### 脚本配置\n\n> 下载脚本\n\n```shell\nmkdir -p /usr/local/sersync/{bin,etc,logs}\ncurl -o /usr/local/sersync/bin/inotify.sh https://raw.githubusercontent.com/Echo21bash/ops_script/master/script/inotify-tools%2Brsync/sersync/bin/inotify.sh\ncurl -o /usr/local/sersync/bin/sersync.sh https://raw.githubusercontent.com/Echo21bash/ops_script/master/script/inotify-tools%2Brsync/sersync/bin/sersync.sh\ncurl -o /usr/local/sersync/bin/rsync.sh https://raw.githubusercontent.com/Echo21bash/ops_script/master/script/inotify-tools%2Brsync/sersync/bin/rsync.sh\ncurl -o /usr/local/sersync/bin/stop.sh https://raw.githubusercontent.com/Echo21bash/ops_script/master/script/inotify-tools%2Brsync/sersync/bin/stop.sh\nchmod -R +x /usr/local/sersync/bin\n```\n\n> 创建配置文件\n\n```shell\n###配置守护进程\ncat > /usr/local/sersync/etc/sersync.conf <<'EOF'\n#########################################通用配置#########################################\n#工作目录一般不修改，是sersync软件的安装目录。\nwork_dir=/usr/local/sersync\n\n#日志目录一般不修改，用于存放运行过程中的日志文件。\nlogs_dir=${work_dir}/logs\n#########################################通用配置#########################################\n\n#########################################监听配置#########################################\n#配置同步目录及其别名，别名用于rsyncd模块下创建目录名，格式[别名=监听目录]，支持多个目录。\nlisten_dir=('file-backup=/data/file' 'db-backup=/data/db' 'img-backup=/data/img')\n\n#监听忽略匹配，仅实时同步生效，用于忽略临时文同步时配置。\n#exclude_file_rule=('/data/file=logs|tmp' '/data/img=.gif')\n#########################################监听配置#########################################\n\n#########################################同步配置#########################################\n#开启启动时首次全量同步，对于非生产业务直接开启，生产业务根据情况开启。\nfull_rsync_first_enable=1\n\n#开启文件实时同步，对于小文件如图片、文档等建议开启，对于大文件如镜像建议关闭。同时考虑实时同步对\n#业务性能的影响酌情开启，\nreal_time_sync_enable=1\n\n#实时同步延时(单位s)，实时同步开启后生效，作为实时同步周期。\nreal_time_sync_delay=60\n\n#实时同步并发数，根据服务器资源配置，默认为10，不建议过高。\n#real_time_sync_parallel_num=10\n\n#实时同步超时时间(s)\nreal_time_rsync_timeout=180\n\n#开启周期性全量同步，定期对整个目录进行同步。完成首次全量同步后，周期性全量同步只是同步变化的数据\nfull_rsync_enable=1\n\n#周期性全量同步多线程数量，默认是CPU等于核数。\n#full_rsync_parallel_rsync_num=10\n\n#全量同步超时时间，超过该时间后将推出同步进程，单位是小时（h）\nfull_rsync_timeout=12\n\n#全量同步cron表达式，配置格式同linux定时任务cron时间格式。\ncron_exp=\"05 23 * * *\"\n\n#rsyncd模块与监听目录备份关系，格式[模式名=监听目录,监听目录]一个模式可以对应多个待同步目录逗号\n#分隔，要求模式名称唯一，一个待同步目录只能对应一个模块，否则第一个生效。\nrsyncd_mod=('backup1=/data/file,/data/db' 'backup2=/data/img')\n\n#模块所在主机地址支持多个地址使用逗号分隔，多个地址实现多份备份。\nrsyncd_ip=('backup1=127.0.0.1:873' 'backup2=127.0.0.1:873,192.168.0.163:873')\n\n#rsync同步的用户\nrsync_user=rsync\n\n#rsync密码文件\nrsync_passwd_file=/etc/rsync.passwd\n\n#单个rsync进程传输限速，总的带宽速率为并发数量乘以单个进程带宽限速。\nrsync_bwlimit=10M\n\n#rsync额外参数，建议开启--partial、--append-verify、--ignore-missing-args，当前已开启必要参数\n#-rlptDR --delete\nextra_rsync_args=\"-v --partial --append-verify --ignore-missing-args --contimeout 120\"\n\n#保留多少天内的历史备份(单位d)，防止误删源文件导致数据丢失，备份目录为rsynd模块下/history-backup/，\n#建议大于全量同步周期cron_exp配置的时间否则在未开启实时同步时，数据有误删除无法恢复的风险。使用rsync\n#的--backup和--backup-dir参数实现\nkeep_history_backup_days=7\n#########################################同步配置#########################################\n\n#########################################其他配置#########################################\n#开启额外脚本执行,全量同步前会执行，可用于备份数据库、生成逻辑备份文件、清除无用文件等操作\nexec_command_enable=0\n#监听目录对应的脚本名称\n#exec_command_list=('/data/file=clean_file.sh' '/data/img=clean_img.sh')\n\n#promethues metrics目录，用于node-exporter获取监控指标\ntextfile_collector_dir=/usr/local/node_exporter/textfile_collector\n#########################################其他配置#########################################\nEOF\n```\n\n> 配置守护进程\n\n```shell\ncat > /etc/systemd/system/sersync.service < 启动同步\n\n```shell\n#启动同步\nsystemctl daemon-reload\nsystemctl start sersync\n```\n\n### 容器部署\n\n#### 创建配置\n\n```shell\n#认证文件\necho 'Ki13W@yYZvbJ' >/etc/rsync.passwd\nchmod 600 /etc/rsync.passwd\n```\n\n```shell\n#创建配置文件\nmkdir -p /usr/local/sersync/{bin,etc,logs}\ncat > /usr/local/sersync/etc/sersync.conf <<'EOF'\n#########################################通用配置#########################################\n#工作目录一般不修改，是sersync软件的安装目录。\nwork_dir=/usr/local/sersync\n\n#日志目录一般不修改，用于存放运行过程中的日志文件。\nlogs_dir=${work_dir}/logs\n#########################################通用配置#########################################\n\n#########################################监听配置#########################################\n#配置同步目录及其别名，别名用于rsyncd模块下创建目录名，格式[别名=监听目录]，支持多个目录。\nlisten_dir=('file-backup=/data/file' 'db-backup=/data/db' 'img-backup=/data/img')\n\n#监听忽略匹配，仅实时同步生效，用于忽略临时文同步时配置。\n#exclude_file_rule=('/data/file=logs|tmp' '/data/img=.gif')\n#########################################监听配置#########################################\n\n#########################################同步配置#########################################\n#开启启动时首次全量同步，对于非生产业务直接开启，生产业务根据情况开启。\nfull_rsync_first_enable=1\n\n#开启文件实时同步，对于小文件如图片、文档等建议开启，对于大文件如镜像建议关闭。同时考虑实时同步对\n#业务性能的影响酌情开启，\nreal_time_sync_enable=1\n\n#实时同步延时(单位s)，实时同步开启后生效，作为实时同步周期。\nreal_time_sync_delay=60\n\n#实时同步并发数，根据服务器资源配置，默认为10，不建议过高。\n#real_time_sync_parallel_num=10\n\n#实时同步超时时间(s)\nreal_time_rsync_timeout=180\n\n#开启周期性全量同步，定期对整个目录进行同步。完成首次全量同步后，周期性全量同步只是同步变化的数据\nfull_rsync_enable=1\n\n#周期性全量同步多线程数量，默认是CPU等于核数。\n#full_rsync_parallel_rsync_num=10\n\n#全量同步超时时间，超过该时间后将推出同步进程，单位是小时（h）\nfull_rsync_timeout=12\n\n#全量同步cron表达式，配置格式同linux定时任务cron时间格式。\ncron_exp=\"05 23 * * *\"\n\n#rsyncd模块与监听目录备份关系，格式[模式名=监听目录,监听目录]一个模式可以对应多个待同步目录逗号\n#分隔，要求模式名称唯一，一个待同步目录只能对应一个模块，否则第一个生效。\nrsyncd_mod=('backup1=/data/file,/data/db' 'backup2=/data/img')\n\n#模块所在主机地址支持多个地址使用逗号分隔，多个地址实现多份备份。\nrsyncd_ip=('backup1=127.0.0.1:873' 'backup2=127.0.0.1:873,192.168.0.163:873')\n\n#rsync同步的用户\nrsync_user=rsync\n\n#rsync密码文件\nrsync_passwd_file=/etc/rsync.passwd\n\n#单个rsync进程传输限速，总的带宽速率为并发数量乘以单个进程带宽限速。\nrsync_bwlimit=10M\n\n#rsync额外参数，建议开启--partial、--append-verify、--ignore-missing-args，当前已开启必要参数\n#-rlptDR --delete\nextra_rsync_args=\"-v --partial --append-verify --ignore-missing-args --contimeout 120\"\n\n#保留多少天内的历史备份(单位d)，防止误删源文件导致数据丢失，备份目录为rsynd模块下/history-backup/，\n#建议大于全量同步周期cron_exp配置的时间否则在未开启实时同步时，数据有误删除无法恢复的风险。使用rsync\n#的--backup和--backup-dir参数实现\nkeep_history_backup_days=7\n#########################################同步配置#########################################\n\n#########################################其他配置#########################################\n#开启额外脚本执行,全量同步前会执行，可用于备份数据库、生成逻辑备份文件、清除无用文件等操作\nexec_command_enable=0\n#监听目录对应的脚本名称\n#exec_command_list=('/data/file=clean_file.sh' '/data/img=clean_img.sh')\n\n#promethues metrics目录，用于node-exporter获取监控指标\ntextfile_collector_dir=/usr/local/node_exporter/textfile_collector\n#########################################其他配置#########################################\nEOF\n```\n\n#### 启动容器\n\n```shell\ndocker run --name sersync --privileged -itd -v /data/file:/data/file \\\n-v /data/db:/data/db \\\n-v /etc/rsync.passwd:/etc/rsync.passwd \\\n-v /usr/local/sersync/etc/sersync.conf:/usr/local/sersync/etc/sersync.conf \\\n-v /usr/local/node_exporter/textfile_collector:/usr/local/node_exporter/textfile_collector \\\necho21bash/sersync:1.0\n```\n\n## 容器环境变量\n\n> 默认容器以sersync模式运行，配置文件需要挂载到容器内部。以rsyncd模式运行时可以通过以下环境变量配置也可通过挂载配置文件传入配置。\n\n```shell\n#运行模式sersync为监听同步将文件同步到远程服务器，rsyncd为rsync模式daemon。\nRUN_MODE=${RUN_MODE:-sersync}\n#rsync认证用户\nRSYNCD_USER=${RSYNCD_USER:-rsync}\n#rsync认证密码\nRSYNCD_PASSWD=${RSYNCD_PASSWD:-Ki13W@yYZvbJ}\n#rsyncd模块名称\nRSYNCD_MOD_NAME=${RSYNCD_MOD_NAME:-data}\n#rsyncd存储目录\nRSYNCD_PATH=${RSYNCD_PATH:-/data}\n#rsyncd最大连接数\nRSYNCD_MAX_CONN=${RSYNCD_MAX_CONN:-300}\n#rsyncd白名单\nRSYNCD_HOSTS_ALLOW=${RSYNCD_HOSTS_ALLOW:-0.0.0.0/0}\n```\n\n## 监控指标\n\n> 默认会在配置文件中textfile_collector_dir配置项的目录生成同步的promethues监控指标，需要将指标目录挂载到node-exporter可以读取到的目录，即可通过node-exporter采集到监控指标。\n\n## 其他说明\n\n> 对于目录深度深、文件数量众多的场景，inotify监听启动时间较长；\n\n## 变更记录\n\n2022-08-12\n\n* 解决带空格文件同步错误的问题\n* 解决flock锁文件名过长的问题\n* 增加必要日志\n\n2023-03-23\n\n* 解决带特殊字符文件同步错误的问题\n* 分离配置文件\n* 脚本优化\n\n2023-05-31\n\n* 更换ubuntu为基础镜像\n* 新增执行额外脚本功能\n\n2024-02-29\n\n* 优化脚本\n* 新增promethues监控指标\n","has_starred","hub_user","immutable_tags_settings",{"_71":11,"_72":73},"enabled","rules",[74],".*","is_automated","permissions",{"_78":17,"_79":11,"_80":11},"read","write","admin","user","storage_size",1043820960,"orgData","recentTags",[87],{"_88":40,"_89":90,"_91":92,"_93":94,"_95":96,"_97":98,"_46":114,"_115":90,"_116":29,"_117":52,"_53":118,"_119":120,"_121":105,"_122":114,"_123":62,"_124":17},"content_type","creator",20291750,"digest","sha256:b132872dd72d7f6f9e25c2d229033d2596ecf51924e674c9c929628dbc31fa5e","full_size",116121567,"id",421101414,"images",[99],{"_100":101,"_91":92,"_102":103,"_104":105,"_106":107,"_108":109,"_110":103,"_111":-5,"_112":94,"_59":62,"_113":-5},"architecture","amd64","features","","last_pulled","2026-09-12T23:20:17.333907238Z","last_pushed","2024-03-25T07:11:23Z","os","linux","os_features","os_version","size","variant","2024-03-25T07:11:22.33395Z","last_updater","last_updater_username","media_type","1.0","repository",19736990,"tag_last_pulled","tag_last_pushed","tag_status","v2","sbxKitMetadata","isPotentialSpam","isHubSbxKitsEnabled","showSbxLogoOverride","sbxTrademarkText","displaySecurityMetadata","tagAttestations","tagsHaveAttestations",[],"routes/_layout.r.$namespace.$reponame.tags",{"_34":136,"_143":144,"_132":153,"_154":11},{"_36":137,"_38":138,"_41":42,"_43":44,"_45":11,"_46":47,"_48":49,"_50":139,"_53":31,"_28":29,"_54":55,"_56":40,"_57":58,"_59":60,"_61":62,"_63":-5,"_64":58,"_65":66,"_67":11,"_68":29,"_69":140,"_75":11,"_76":142,"_81":29,"_82":83},[],[40],[52],{"_71":11,"_72":141},[74],{"_78":17,"_79":11,"_80":11},"tags",{"_145":60,"_146":-5,"_147":-5,"_148":149},"count","next","previous","results",[150],{"_88":40,"_89":90,"_91":92,"_93":94,"_95":96,"_97":151,"_46":114,"_115":90,"_116":29,"_117":52,"_53":118,"_119":120,"_121":105,"_122":114,"_123":62,"_124":17},[152],{"_100":101,"_91":92,"_102":103,"_104":105,"_106":107,"_108":109,"_110":103,"_111":-5,"_112":94,"_59":62,"_113":-5},[],"canDeleteTags","actionData","errors"]

echo21bash/sersync

Tags for echo21bash/sersync