虚拟机搭建 k8s 集群实验环境及 Harbor 私有仓库

answer VNET_1_VIRTUAL_ADAPTER yes
answer VNET_4_DHCP yes
answer VNET_4_HOSTONLY_NETMASK 255.255.255.0
answer VNET_4_HOSTONLY_SUBNET 192.168.8.0
answer VNET_4_NAT yes
answer VNET_4_NAT_PARAM_UDP_TIMEOUT 30
answer VNET_4_VIRTUAL_ADAPTER yes
answer VNET_8_DHCP yes
复制代码

# NAT gateway address
ip = 192.168.8.2
netmask = 255.255.255.0
复制代码

TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=static       # 修改本行，默认值为dhcp，改为static
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens33
UUID=b4bd9a95-905e-471a-9313-f7a668a8af5d
DEVICE=ens33
ONBOOT=yes            # 修改本行，默认值为no，改为yes
IPADDR=192.168.8.10   # 新增本行
NETMASK=255.255.255.0 # 新增本行
GATEWAY=192.168.8.2   # 新增本行，这是前面vnet2的gateway ip
DNS1=114.114.114.114  # 新增本行
复制代码

#在不同主机设置主机名
hostnamectl set-hostname k8s-master-1
hostnamectl set-hostname k8s-node-1
hostnamectl set-hostname k8s-node-2

scp /etc/hosts root@k8s-node-01:/ect/hosts
scp /etc/hosts root@k8s-node-02:/ect/hosts

yum install -y conntrack ntpdate ntp ipvsadm ipset jq iptables curl sysstat libseccomp wget vim net-tools git

#备份yum源
mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo_bak
#下载阿里yum源
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
#更新yum缓存
yum clean all
yum makecache

#关闭防火墙 firewalld 
systemctl stop firewalld && systemctl disable firewalld

#下载iptables
yum install -y iptables-services  && systemctl start iptables && systemctl enable iptables  && iptables -F && service iptables save

swapoff -a && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config

cat > kubernetes.conf <<EOF
net.bridge.bridge-nf-call-iptables=1 # 这两条是开启网桥模式
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
vm.swappiness=0 # 禁止使用 swap 空间，只有当系统 OOM 时才允许使用它
vm.overcommit_memory=1 # 不检查物理内存是否够用
vm.panic_on_oom=0 # 开启 OOM
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1 # 关闭 ipv6 协议
net.netfilter.nf_conntrack_max=2310720
EOF

# 设置系统时区为 中国/上海
timedatectl set-timezone Asia/Shanghai
# 将当前的 UTC 时间写入硬件时钟
timedatectl set-local-rtc 0
# 重启依赖于系统时间的服务

systemctl restart rsyslog
systemctl restart crond

mkdir /var/log/journal # 持久化保存日志的目录
mkdir /etc/systemd/journald.conf.d
cat > /etc/systemd/journald.conf.d/99-prophet.conf <<EOF
[Journal]

# 持久化保存到磁盘

Storage=persistent
# 压缩历史日志
Compress=yes
SyncIntervalSec=5m
RateLimitInterval=30s
RateLimitBurst=1000
# 最大占用空间 10G
SystemMaxUse=10G
# 单日志文件最大 200M
SystemMaxFileSize=200M
# 日志保存时间 2 周
MaxRetentionSec=2week

# 不将日志转发到 syslog
ForwardToSyslog=no
EOF

systemctl restart systemd-journald

rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
# 安装完成后检查 /boot/grub2/grub.cfg 中对应内核 menuentry 中是否包含 initrd16 配置，如果没有，再安装
一次！

grub2-set-default 'CentOS Linux (4.4.189-1.el7.elrepo.x86_64) 7 (Core)'

reboot

modprobe br_netfilter

cat > /etc/sysconfig/modules/ipvs.modules <<EOF

#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4

EOF

chmod 755 /etc/sysconfig/modules/ipvs.modules  && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4

yum install -y yum-utils device-mapper-persistent-data lvm2

yum-config-manager \
--add-repo \
http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

yum update -y  && yum install -y docker-ce

# 创建 /etc/docker 目录
mkdir /etc/docker

# 配置 daemon.
insecure-registries  允许Docker使用无权限验证的 Docker Registry 即私有镜像仓库 Harbor

cat > /etc/docker/daemon.json <<EOF
{
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"registry-mirrors": ["这里可以写你的阿里云镜像加速地址"],
"log-opts": {
"max-size": "100m"
},
"insecure-registries": ["hub.czp.com"]
}
EOF

mkdir -p /etc/systemd/system/docker.service.d

# 重启docker服务

systemctl daemon-reload && systemctl restart docker && systemctl enable docker

cat  > /etc/yum.repos.d/kubernetes.repo <<EOF
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
yum -y install kubeadm-1.15.1 kubectl-1.15.1 kubelet-1.15.1
systemctl enable kubelet.service

# 生成kubeadm-config配置文件
kubeadm config print init-defaults > kubeadm-config.yaml

# 然后打开这个文件，修改几个参数
vim kubeadm-config.yaml

imageRepository: registry.aliyuncs.com/google_containers #修改下镜像地址
advertiseAddress: 192.168.8.10 # masterip
kubernetesVersion: v1.15.1 # kubeadmin版本
networking:
podSubnet: "10.244.0.0/16" # 必须是这个ip
serviceSubnet: 10.96.0.0/12

# 然后在末尾添加如下内容，支持ipvs
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
featureGates:
  SupportIPVSProxyMode: true
mode: ipvs

# 开始初始化
kubeadm init --config=kubeadm-config.yaml --experimental-upload-certs | tee kubeadm-init.log


#查看是否初始化成功
grep "initialized successfully" kubeadm-init.log 


#初始化成功后的后续操作
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

#把初始化的配置和日志保存起来
mkdir install-k8s 
mv  kubeadm-init.log   kubeadm-config.yaml install-k8s


wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

kubectl apply -f kube-flannel.yml

# 我在这边遇到了一个问题。quay.io/coreos/flannel 镜像一直无法正常 pull，我是在 https://github.com/coreos/flannel/releases 中下载后，上传到虚拟机，安装手动 load 到 docker 中。
scp ./flanneld-v0.13.1-rc1-amd64.docker root@192.168.8.21:/root
docker load < flanneld-v0.13.1-rc1-amd64.docker

#查看 node 状态
kubectl get node

kubeadm join 192.168.8.10:6443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:1d853fdcfd4c933dee2c13b8e5ee7ac91fd85225d4c6f96f05d9a1997591012c

kubectl get node

NAME           STATUS   ROLES    AGE   VERSION
k8s-master01   Ready    master   77m   v1.15.1
k8s-node-1     Ready    <none>   69m   v1.15.1

yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager \
--add-repo \
http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum update -y  && yum install -y docker-ce

# 创建 /etc/docker 目录
mkdir /etc/docker

# 配置 daemon. insecure-registries 是后续用来访问 harbor 私有仓库的, 表示可信任的仓库地址

cat > /etc/docker/daemon.json <<EOF
{
"exec-opts": "native.cgroupdriver=systemd",
"log-driver": "json-file",
"registry-mirrors": "https://tosnxdv7.mirror.aliyuncs.com",
"log-opts": {
"max-size": "100m"
}
"insecure-registries": "hub.czp.com"
}
EOF
mkdir -p /etc/systemd/system/docker.service.d

# 重启docker服务
systemctl daemon-reload && systemctl restart docker && systemctl enable docker

wget https://linux-soft-ware.oss-cn-shenzhen.aliyuncs.com/docker-compose
chmod +x docker-compose
mv docker-compose /usr/local/bin

下载地址：https://github.com/goharbor/harbor/releases 这里我使用的是 2.1.0 版本

# 解压缩
tar xf harbor-offline-installer-v2.1.0.tgz

# 编辑配置文件
vi harbor.yml

# 这里我们需要修改 2 个地方
# 1，设置域名,这里要跟之前的 docker/daemon.json 里的 insecure-registries 保持一致。
hostname = hub.czp.com

# 2，设置 https 的密钥和证书
https:
  # https port for harbor, default is 443
  port: 443
  # The path of cert and key files for nginx
  certificate: /data/cert/server.crt
  private_key: /data/cert/server.key

mkdir /data/cert -p

# 1、创建密钥

openssl genrsa -des3 -out server.key 2048


Generating RSA private key, 2048 bit long modulus
............................................................+++
..............................................+++
e is 65537 (0x10001)
Enter pass phrase for server.key: 输入密码 123456
Verifying - Enter pass phrase for server.key:	输入密码 123456

2、创建证书

openssl req -new -key server.key -out server.csr

Enter pass phrase for server.key:
You are about to be asked to enter information that will be incorporated
into your certificate request.
What you are about to enter is what is called a Distinguished Name or a DN.
There are quite a few fields but you can leave some blank
For some fields there will be a default value,
If you enter '.', the field will be left blank.
-----
Country Name (2 letter code) [XX]:CN
State or Province Name (full name) []:GD
Locality Name (eg, city) [Default City]:SZ
Organization Name (eg, company) [Default Company Ltd]:DEMO
Organizational Unit Name (eg, section) []:DEMO
Common Name (eg, your name or your server's hostname) []:hub.czp.com
Email Address []:ASDASD@163.COM

Please enter the following 'extra' attributes
to be sent with your certificate request
A challenge password []:
An optional company name []:


3、让证书退去密码
openssl rsa -in server.key.org -out server.key


Enter pass phrase for server.key.org:输入密码 123456
writing RSA key


4、签名
openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt


5、修改证书权限
chmod -R 777 /data/cert

./harbor/install.sh

10.0.0.11  hub.czp.com

# 访问 harbor

输入 hub.czp.com, 输入 admin/12345 ![](https://p1-juejin.byteimg.com/tos-cn-i-k3u1fbpfcp/10d8c40ecd014109acfbe875b557e1aa~tplv-k3u1fbpfcp-watermark.image)

 docker login https://hub.czp.com
 
 Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded