本测试记录从openshift 3.6环境中导出项目，然后在将项目环境恢复到Openshift 3.11中所需要的步骤
从而指导导入导出的升级过程。
1.安装Openshift 3.6版本
过程略
2.安装OpenShift 3.11版本
过程略
3.在Openshift 3.6版本中建立各类资源
- 创建用户
htpasswd /etc/origin/master/htpasswd eric htpasswd /etc/origin/master/htpasswd alice
- 给节点打标签
oc label node node2.example.com application=eric-tomcat
[root@master ~]# oc get node node2.example.com --show-labels
NAME STATUS AGE VERSION LABELS
node2.example.com Ready 1d v1.6.1+5115d708d7 application=eric-tomcat,beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=node2.example.com,region=infra,zone=default
- 导入镜像
docker load -i tomcat.tar
docker tag docker.io/tomcat:-slim registry.example.com/tomcat:-slim
docker push registry.example.com/tomcat:-slim
- 创建项目ericproject1
用eric用户登录
oc new-project ericproject1
oc import-image tomcat:-slim --from=registry.example.com/tomcat:-slim --insecure --confirm
oc new-app tomcat:-slim --name=ericapp1
oc expose service ericapp1
oc scale dc/ericapp1 --replicas=3
oc new-app tomcat:8-slim --name=ericapp2
oc expose service ericapp2
- 创建项目ericproject2
用eric用户登录
oc new-project ericproject2
oc import-image tomcat:-slim --from=registry.example.com/tomcat:-slim --insecure --confirm
oc new-app tomcat:-slim --name=eric-tomcat
oc expose service eric-tomcat
- 建立template
[root@master ~]# cat eric2tomcat-project2.yaml
apiVersion: v1
kind: Template
metadata:
creationTimestamp: null
name: eric2tomcat
objects:
- apiVersion: v1
kind: DeploymentConfig
metadata:
annotations:
openshift.io/generated-by: OpenShiftNewApp
creationTimestamp: null
generation:
labels:
app: ${APP_NAME}
name: ${APP_NAME}
spec:
replicas:
selector:
app: ${APP_NAME}
deploymentconfig: ${APP_NAME}
strategy:
activeDeadlineSeconds:
resources: {}
rollingParams:
intervalSeconds:
maxSurge: %
maxUnavailable: %
timeoutSeconds:
updatePeriodSeconds:
type: Rolling
template:
metadata:
annotations:
openshift.io/generated-by: OpenShiftNewApp
creationTimestamp: null
labels:
app: ${APP_NAME}
deploymentconfig: ${APP_NAME}
spec:
containers:
- image: registry.example.com/tomcat@sha256:8f701fff708316aabc01520677446463281b5347ba1d6e9e237dd21de600f809
imagePullPolicy: IfNotPresent
name: ${APP_NAME}
ports:
- containerPort:
protocol: TCP
resources: {}
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
terminationGracePeriodSeconds:
test: false
triggers:
- type: ConfigChange
- imageChangeParams:
automatic: true
containerNames:
- ${APP_NAME}
from:
kind: ImageStreamTag
name: tomcat:-slim
namespace: ericproject2
type: ImageChange
status:
availableReplicas:
latestVersion:
observedGeneration:
replicas:
unavailableReplicas:
updatedReplicas:
- apiVersion: v1
kind: Service
metadata:
annotations:
openshift.io/generated-by: OpenShiftNewApp
creationTimestamp: null
labels:
app: ${APP_NAME}
name: ${APP_NAME}
spec:
ports:
- name: -tcp
port:
protocol: TCP
targetPort:
selector:
app: ${APP_NAME}
deploymentconfig: ${APP_NAME}
sessionAffinity: None
type: ClusterIP
status:
loadBalancer: {}
- apiVersion: v1
kind: Route
metadata:
annotations:
openshift.io/host.generated: "true"
creationTimestamp: null
labels:
app: ${APP_NAME}
name: ${APP_NAME}
spec:
host: ${APP_NAME}-ericproject2.app.example.com
port:
targetPort: -tcp
to:
kind: Service
name: ${APP_NAME}
weight:
wildcardPolicy: None
status:
ingress:
- conditions:
- lastTransitionTime: --07T15::35Z
status: "True"
type: Admitted
host: ${APP_NAME}-ericproject2.app.example.com
routerName: router
wildcardPolicy: None
parameters:
- name: APP_NAME
displayname: application name
value: myapp
oc create -f eric2tomcat-project2.yaml
- 基于template建立应用
oc new-app eric2tomcat
- 建立configmap
[root@master ~]# cat nginx.conf
user nginx;
worker_processes ; error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid; events {
worker_connections ;
} http {
include /etc/nginx/mime.types;
default_type application/octet-stream; log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"'; access_log /var/log/nginx/access.log main; sendfile on;
#tcp_nopush on; keepalive_timeout ; #gzip on; include /etc/nginx/conf.d/*.conf;
}
oc create configmap nginx-conf --from-file=nginx.conf
- gluster pv相关设置
[root@master ~]# cat gluster-endpoints.yaml
apiVersion: v1
kind: Endpoints
metadata:
name: gluster-endpoints
subsets:
- addresses:
- ip: 192.168.56.107
ports:
- port:
protocol: TCP
- addresses:
- ip: 192.168.56.108
ports:
- port:
protocol: TCP
[root@master ~]# cat gluster-service.yaml
apiVersion: v1
kind: Service
metadata:
name: gluster-service
spec:
ports:
- port:
[root@master ~]# cat gluster-pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: gluster-pv
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteMany
glusterfs:
endpoints: gluster-endpoints
path: /gv0
readOnly: false
persistentVolumeReclaimPolicy: Retain
[root@master ~]# cat tomcat-claim.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: tomcat-claim
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1Gi
- 用alice账户登录创建项目
oc new-project alice-project
oc import-image tomcat:-slim --from=registry.example.com/tomcat:-slim --insecure --confirm
oc new-app tomcat:-slim --name=alice-tomcat
oc expose service alice-tomcat
oc scale dc/alice-tomcat --replicas=
4.开始资源导出过程
以下在OpenShift 3.6的集群环境下操作。
先下载jq和安装(在执行导出的集群的节点和执行导入的集群节点上都需要安装)
https://stedolan.github.io/jq/
执行导出
./project_export.sh ericproject1
./project_export.sh ericproject2
./project_export.sh alice-project
导出完成后发现当前目录下有这三个目录
导出后进入项目查看内容
简单写了个批量导出项目的脚本
[root@master ~]# cat all_export.sh result="true";
systemproject=(kube-system kube-public kube-service-catalog default logging management-infra openshift openshift-infra) for i in $(oc get projects | awk 'NR>1{print $1}'); do
# echo $i
for j in ${systemproject[@]}; do
# echo $j
if [ $i == $j ]; then
# echo "enter"
result="false"
fi;
done
if [ $result == "true" ]; then
echo $i;
./project_export.sh $i;
fi;
result="true"
done
导出截取了一段ericproject2
ericproject2
###########
# WARNING #
###########
This script is distributed WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND
Beware ImageStreams objects are not importables due to the way they work
See https://github.com/openshift/openshift-ansible-contrib/issues/967
for more information
Exporting namespace to ericproject2/ns.json
Exporting 'rolebindings' resources to ericproject2/rolebindings.json
Exporting 'serviceaccounts' resources to ericproject2/serviceaccounts.json
Exporting 'secrets' resources to ericproject2/secrets.json
Exporting deploymentconfigs to ericproject2/dc_*.json
Patching DC...
Patching DC...
Exporting 'bc' resources to ericproject2/bcs.json
Skipped: list empty
Exporting 'builds' resources to ericproject2/builds.json
Skipped: list empty
Exporting 'is' resources to ericproject2/iss.json
Exporting 'imagestreamtags' resources to ericproject2/imagestreamtags.json
Exporting 'rc' resources to ericproject2/rcs.json
Exporting services to ericproject2/svc_*.json
Exporting 'po' resources to ericproject2/pods.json
Exporting 'podpreset' resources to ericproject2/podpreset.json
the server doesn't have a resource type "podpreset"
Skipped: no data
Exporting 'cm' resources to ericproject2/cms.json
Exporting 'egressnetworkpolicies' resources to ericproject2/egressnetworkpolicies.json
Skipped: list empty
Exporting 'rolebindingrestrictions' resources to ericproject2/rolebindingrestrictions.json
Skipped: list empty
Exporting 'cm' resources to ericproject2/cms.json
Exporting 'egressnetworkpolicies' resources to ericproject2/egressnetworkpolicies.json
Skipped: list empty
Exporting 'rolebindingrestrictions' resources to ericproject2/rolebindingrestrictions.json
Skipped: list empty
Exporting 'limitranges' resources to ericproject2/limitranges.json
Skipped: list empty
Exporting 'resourcequotas' resources to ericproject2/resourcequotas.json
Skipped: list empty
Exporting 'pvc' resources to ericproject2/pvcs.json
Skipped: list empty
Exporting 'pvc' resources to ericproject2/pvcs_attachment.json
Skipped: list empty
Exporting 'routes' resources to ericproject2/routes.json
Exporting 'templates' resources to ericproject2/templates.json
Exporting 'cronjobs' resources to ericproject2/cronjobs.json
Skipped: list empty
Exporting 'statefulsets' resources to ericproject2/statefulsets.json
Skipped: list empty
Exporting 'hpa' resources to ericproject2/hpas.json
Skipped: list empty
Exporting 'deploy' resources to ericproject2/deployments.json
Skipped: list empty
Exporting 'replicasets' resources to ericproject2/replicasets.json
Skipped: list empty
Exporting 'poddisruptionbudget' resources to ericproject2/poddisruptionbudget.json
Skipped: list empty
Exporting 'daemonset' resources to ericproject2/daemonset.json
Skipped: list empty
5.执行导入过程
将三个目录全部拷贝到执行导入的节点，OpenShift 3.11的版本
- 先导入镜像
docker load -i tomcat.tar
docker tag docker.io/tomcat:-slim registry.example.com/tomcat:-slim
docker push registry.example.com/tomcat:-slim
- 以admin的身份登录，然后运行
./project_import.sh ericproject1
./project_import.sh ericproject2
./project_import.sh alice-project
6. 恢复到3.11后的验证
- 用户
[root@master ~]# oc get users
NAME UID FULL NAME IDENTITIES
admin 3d7951e7-422a-11e9-90df-080027dc991a htpasswd_auth:admin
可见导入过程并不会对用户进行任何操作，但实际环境中openshift集群都是连接LDAP或其他外部用户，所以这关系不大。
- 项目
[root@master ~]# oc projects
You have access to the following projects and can switch between them with 'oc project <projectname>': * alice-project
default
ericproject1
ericproject2
kube-public
kube-system
management-infra
openshift
openshift-console
openshift-infra
openshift-logging
openshift-metrics-server
openshift-monitoring
openshift-node
openshift-sdn
openshift-web-console Using project "alice-project" on server "https://master.example.com:8443".
通过admin能看到所有的导入项目，进入项目后因为image stream的问题，发现有些DeploymentConfig一直在deploy阶段，但并无实例运行
运行下面的命令让实例重新装载
oc delete pod alice-tomcat--deploy
oc rollout latest alice-tomcat
然后就可以看到实例全部装载成功
- label
可见并没有将我们的label导入到新环境中
[root@master ~]# oc get nodes --show-labels
NAME STATUS ROLES AGE VERSION LABELS
master.example.com Ready master 2d v1.11.0+d4cacc0 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=master.example.com,node-role.kubernetes.io/master=true
node1.example.com Ready infra 2d v1.11.0+d4cacc0 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=node1.example.com,node-role.kubernetes.io/infra=true
node2.example.com Ready compute 2d v1.11.0+d4cacc0 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=node2.example.com,node-role.kubernetes.io/compute=true
- 权限RBAC
[root@master ~]# oc get rolebinding
NAME ROLE USERS GROUPS SERVICE ACCOUNTS SUBJECTS
admin /admin alice
system:deployers /system:deployer deployer
system:image-builders /system:image-builder builder
system:image-pullers /system:image-puller system:serviceaccounts:alice-project
[root@master ~]# oc project ericproject1
Now using project "ericproject1" on server "https://master.example.com:8443".
[root@master ~]# oc get rolebinding
NAME ROLE USERS GROUPS SERVICE ACCOUNTS SUBJECTS
admin /admin eric
system:deployers /system:deployer deployer
system:image-builders /system:image-builder builder
system:image-pullers /system:image-puller system:serviceaccounts:ericproject1
[root@master ~]# oc project ericproject2
Now using project "ericproject2" on server "https://master.example.com:8443".
[root@master ~]# oc get rolebinding
NAME ROLE USERS GROUPS SERVICE ACCOUNTS SUBJECTS
admin /admin eric
system:deployers /system:deployer deployer
system:image-builders /system:image-builder builder
system:image-pullers /system:image-puller system:serviceaccounts:ericproject2
[root@master ~]#
可见所有的项目权限都保存下来。
7.升级建议
因为原有的集群下节点数目和新的集群很可能不一样，因此单纯的备份etcd和恢复etcd的办法上有很大风险。
这种模式下，采用项目导入导出的方式不失为一种较为安全的方式。
需要注意的地方包括:
- 用户不会导出，但在openshift的权限信息会保存。
- 节点的Label不会导出
- 导入导出过程需要rollout。
- 用glusterfs的时候，每个project的gluster-endpoint资源没有保存下来，估计和gluster-service没有关联相关
- 因为pv不是属于项目资源而属于整个集群资源，导入项目前，先建立pv
- 遇到pod无法启动很多时候和mount存储有关系