registry 私有镜像仓库 k8s 高可用

####registry 私有镜像仓库 k8s 高可用

####################################################
####################################################所有节点执行

###所有节点执行
##每一个节点安装GlusterFS
yum install -y centos-release-gluster
yum install glusterfs-server -y

#配置 GlusterFS 集群：
#启动 glusterFS

systemctl restart glusterd.service
systemctl enable glusterd.service


#创建数据存储目录：
mkdir -p /gfs1

####在 swarm-manager 节点上配置，将 节点 加入到 集群中。
##gluster peer probe hostname
####################################################
####################################################只在主节点
gluster peer probe node224
gluster peer probe node225


###查看集群状态：
gluster peer status



###所有节点执行
##创建GlusterFS磁盘： 复制模式

gluster volume create gv1 replica 3 transport tcp node223:/gfs1 node224:/gfs1 node225:/gfs1 force

#启动 gv1
gluster volume start gv1

###再查看 volume 状态：
gluster volume info gv1

####################################################
####################################################客户端挂载volume  所有节点执行

mkdir -p /gv1
mount -t glusterfs localhost:gv1 /gv1
echo 'localhost:/gv1 /gv1 glusterfs _netdev,rw,acl 0 0' >>/etc/fstab



####################################################
#################################################### 生成证书  只在主节点操作

mkdir -p /gv1/registry/{certs,registry}

yum install -y expect openssl

####创建证书
expect -c  '
spawn  openssl req -newkey rsa:4096 -nodes -sha256 -keyout /gv1/registry/certs/domain.key -x509 -days 3650 -out /gv1/registry/certs/domain.crt
expect {
    "Country Name " { send "cn\r"; exp_continue}
    "State or Province Name" { send "sc\r" ; exp_continue}
    "Locality Name " { send "cd\r"; exp_continue}
    "Default Company Ltd" { send "k8s\r"; exp_continue}
    "Organizational Unit Name" { send "sys\r"; exp_continue}
    "Common Name " { send "k.xxxx.com\r" ; exp_continue}
    "Email Address " { send "\r" ; exp_continue}
    eof { exit }
}'


mkdir mkdir -p /etc/docker/certs.d/k.xxxx.com:30443
\cp /gv1/registry/certs/domain.crt /etc/docker/certs.d/k.xxxx.com:30443/ca.crt
####################################################
####################################################利用k8s启动镜像仓库 只在主节点操作

cat >registry.yaml <<EOF
apiVersion: v1
kind: ReplicationController
metadata:
  name: registry
  namespace: kube-system
spec:
  replicas: 1
  selector:
    app: registry
  template:
    metadata:
      labels:
        app: registry
    spec:
      containers:
        - name: registry
          image: registry:2
          ports:
            - containerPort: 5000
          env:
            - name: REGISTRY_HTTP_TLS_CERTIFICATE
              value: "/certs/domain.crt"
            - name: REGISTRY_HTTP_TLS_KEY
              value: "/certs/domain.key"
          volumeMounts:
            - name: registry
              mountPath: /var/lib/registry
            - name: certs
              mountPath: /certs
      volumes:
        - name: registry
          hostPath:
            path: /gv1/registry/registry
        - name: certs
          hostPath:
            path: /gv1/registry/certs

---
apiVersion: v1
kind: Service
metadata:
 name: registry
 namespace: kube-system
spec:
  type: NodePort
  ports:
    - port: 6000
      targetPort: 5000
      nodePort: 30443
  selector:
    app: registry
EOF

kubectl apply -f registry.yaml

#kubectl delete -f registry.yaml

curl --cacert /gv1/registry/certs/domain.crt https://k.xxxx.com:30443/v2/_catalog

kubectl get pods,svc -n kube-system |grep registry

####################################################
####################################################不用k8s启动镜像仓库，也可以手动运行docker容器(需要几个节点都操作)
####
mkdir mkdir -p /etc/docker/certs.d/k.xxxx.com:5000
\cp /gv1/registry/certs/domain.crt /etc/docker/certs.d/k.xxxx.com:5000/ca.crt

docker run -d --restart=always --privileged=true --name registry -p 5000:5000 -v /gv1/registry/certs:/certs -v /gv1/registry/registry:/var/lib/registry -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt -e REGISTRY_HTTP_TLS_KEY=/certs/domain.key docker.io/registry:2

#####