kubevirt在360的探索之路（K8s接管虚拟化），涨姿势~

type DomainManager interface { //SyncVMI 为创建虚拟机SyncVMI(*v1.VirtualMachineInstance, bool, *cmdv1.VirtualMachineOptions) (*api.DomainSpec, error)//暂停VMIPauseVMI(*v1.VirtualMachineInstance) error//恢复暂停的VMIUnpauseVMI(*v1.VirtualMachineInstance) errorKillVMI(*v1.VirtualMachineInstance) error//删除VMIDeleteVMI(*v1.VirtualMachineInstance) errorSignalShutdownVMI(*v1.VirtualMachineInstance) errorMarkGracefulShutdownVMI(*v1.VirtualMachineInstance) errorListAllDomains() ([]*api.Domain, error)//迁移VMIMigrateVMI(*v1.VirtualMachineInstance, *cmdclient.MigrationOptions) errorPrepareMigrationTarget(*v1.VirtualMachineInstance, bool) errorGetDomainStats() ([]*stats.DomainStats, error)//取消迁移CancelVMIMigration(*v1.VirtualMachineInstance) error//如下需要启用Qemu guest agent，没启用会包VMI does not have guest agent connectedGetGuestInfo() (v1.VirtualMachineInstanceGuestAgentInfo, error)GetUsers() ([]v1.VirtualMachineInstanceGuestOSUser, error)GetFilesystems() ([]v1.VirtualMachineInstanceFileSystem, error)SetGuestTime(*v1.VirtualMachineInstance) error}

[root@openstack825 ~]# kubectl get vmi -o wideNAME                          AGE     PHASE     IP             NODENAME        LIVE-MIGRATABLEtest100.foo.demo.example.com     8d   Running   192.168.10.30  10.10.67.244   Truetest200.foo.demo.example.com     8d   Running   192.168.10.31  10.10.67.245   True

获取已安装的kubevirt pod[root@openstack825 ~]# kubectl  -n kubevirt get podNAME                               READY   STATUS    RESTARTS   AGEvirt-api-68c958dd-6sx4n            1/1     Running   0          14dvirt-api-68c958dd-sldgr            1/1     Running   0          14dvirt-controller-647d666bd5-gsnzf   1/1     Running   1          14dvirt-controller-647d666bd5-hshnz   1/1     Running   1          14dvirt-handler-4g7ck                 1/1     Running   3          14dvirt-handler-kzv86                 1/1     Running   0          14dvirt-handler-m2ppb                 1/1     Running   0          14dvirt-handler-v6fgt                 1/1     Running   0          14dvirt-operator-65ccf74f56-b82kz     1/1     Running   0          14dvirt-operator-65ccf74f56-zs2xq     1/1     Running   0          14dvirtvnc-947874d99-hn7k5            1/1     Running   0          6d19h

//主要通过调用磁盘、网络相关函数，来判断当前VMI 是否适合迁移func (d *VirtualMachineController) calculateLiveMigrationCondition(vmi *v1.VirtualMachineInstance, hasHotplug bool) (*v1.VirtualMachineInstanceCondition, bool) {liveMigrationCondition := v1.VirtualMachineInstanceCondition{Type:   v1.VirtualMachineInstanceIsMigratable,Status: k8sv1.ConditionTrue,}//调用 checkvolume 方法isBlockMigration, err := d.checkVolumesForMigration(vmi)if err != nil {//如果返回错误信息则会限制迁移liveMigrationCondition.Status = k8sv1.ConditionFalseliveMigrationCondition.Message = err.Error()liveMigrationCondition.Reason = v1.VirtualMachineInstanceReasonDisksNotMigratablereturn &liveMigrationCondition, isBlockMigration}//调用网络模式检查方法err = d.checkNetworkInterfacesForMigration(vmi)if err != nil {liveMigrationCondition = v1.VirtualMachineInstanceCondition{Type:    v1.VirtualMachineInstanceIsMigratable,Status:  k8sv1.ConditionFalse,Message: err.Error(),Reason:  v1.VirtualMachineInstanceReasonInterfaceNotMigratable,}return &liveMigrationCondition, isBlockMigration}if hasHotplug {liveMigrationCondition = v1.VirtualMachineInstanceCondition{Type:    v1.VirtualMachineInstanceIsMigratable,Status:  k8sv1.ConditionFalse,Message: "VMI has hotplugged disks",Reason:  v1.VirtualMachineInstanceReasonHotplugNotMigratable,}return &liveMigrationCondition, isBlockMigration}return &liveMigrationCondition, isBlockMigration}

//checkvolume 定义/检查所有VMI卷共享可以在源和实时迁移的目的地之间热迁移//当所有卷均已共享且VMI没有本地磁盘时，blockMigrate才返回True//某些磁盘组合使VMI不适合实时迁移， 在这种情况下，将返回相关错误func (d *VirtualMachineController) checkVolumesForMigration(vmi *v1.VirtualMachineInstance) (blockMigrate bool, err error) {for _, volume := range vmi.Spec.Volumes {volSrc := volume.VolumeSourceif volSrc.PersistentVolumeClaim != nil || volSrc.DataVolume != nil {var volName stringif volSrc.PersistentVolumeClaim != nil {volName = volSrc.PersistentVolumeClaim.ClaimName} else {volName = volSrc.DataVolume.Name}//pvcutils.IsSharedPVCFromClient_, shared, err := pvcutils.IsSharedPVCFromClient(d.clientset, vmi.Namespace, volName)if errors.IsNotFound(err) {return blockMigrate, fmt.Errorf("persistentvolumeclaim %v not found", volName)} else if err != nil {return blockMigrate, err}if !shared {return true, fmt.Errorf("cannot migrate VMI with non-shared PVCs")}} else if volSrc.HostDisk != nil {shared := volSrc.HostDisk.Shared != nil && *volSrc.HostDisk.Sharedif !shared {return true, fmt.Errorf("cannot migrate VMI with non-shared HostDisk")}} else {blockMigrate = true}}return}func IsSharedPVCFromClient(client kubecli.KubevirtClient, namespace string, claimName string) (pvc *k8sv1.PersistentVolumeClaim, isShared bool, err error) {pvc, err = client.CoreV1().PersistentVolumeClaims(namespace).Get(claimName, v1.GetOptions{})if err == nil {//IsPVCSharedisShared = IsPVCShared(pvc)}return}//IsPVCShared Shared 判断，函数返回bool 类型，成功则返回truefunc IsPVCShared(pvc *k8sv1.PersistentVolumeClaim) bool {//循环PVC的accessModesfor _, accessMode := range pvc.Spec.AccessModes {if accessMode == k8sv1.ReadWriteMany {return true}}return false}

[root@kubevirt01 ~]# kubectl get podNAME                                               READY   STATUS    RESTARTS   AGEcsi-rbdplugin-7bprd                                3/3     Running   0          14dcsi-rbdplugin-fl5c9                                3/3     Running   0          14dcsi-rbdplugin-ggj9q                                3/3     Running   0          14dcsi-rbdplugin-provisioner-84bb9bdd56-7qtnh         6/6     Running   0          14dcsi-rbdplugin-provisioner-84bb9bdd56-sdscf         6/6     Running   0          14dcsi-rbdplugin-provisioner-84bb9bdd56-xjz2r         6/6     Running   0          14dcsi-rbdplugin-svfv2

---apiVersion: v1kind: PersistentVolumeClaimmetadata: name: testzhangsanlisispec: accessModes:   - ReadWriteMany volumeMode: Block resources:   requests:     storage: 10Gi storageClassName: csi-rbd-sckubectl apply -f pvc-test.yaml[root@kubevirt01 ~]# kubectl get pvcNAME                           STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGEtestzhangsanlisi                       Bound    pvc-2c98391d-a2eb-4dd1-a1f0-2a8ef27d4ca3   10Gi       RWX            csi-rbd-sc     3s

(ceph-mon)[root@example100 /]# ceph -s cluster:   id:     d8ab2087-f55c-4b8f-913d-fc60d6fc455d   health: HEALTH_OK services:   mon: 3 daemons, quorum 192.168.10.100 192.168.20.100 192.168.30.100 (age 4d)   mgr: ceph100(active, since 4d), standbys: ceph200， ceph300   osd: 27 osds: 27 up (since 2w), 27 in (since 2w) data:   pools:   1 pools, 1024 pgs   objects: 55.91k objects, 218 GiB   usage:   682 GiB used, 98 TiB / 98 TiB avail   pgs:     1024 active+clean io:   client:   2.2 KiB/s wr, 0 op/s rd, 0 op/s wr(ceph-mon)[root@example100 /]# ceph dfRAW STORAGE:   CLASS     SIZE       AVAIL      USED        RAW USED     %RAW USED   hdd       98 TiB     98 TiB     655 GiB      682 GiB          0.68   TOTAL     98 TiB     98 TiB     655 GiB      682 GiB          0.68

//Demo Yaml//IP 地址段来自源与网络物理设备分配时spec: cidrBlock: 192.168.10.0/23 default: true excludeIps: - 192.168.10.1 gateway: 192.168.10.1 gatewayNode: "" gatewayType: distributed // 需要设置成false natOutgoing: false private: false protocol: IPv4 provider: ovn //需要设置成true，若为false，会在主机侧加上route，导致net不通 underlayGateway: true vlan: ovn-vlan

[root@kubevirt01 ~]# kubectl -n kube-system get podNAME                                   READY   STATUS    RESTARTS   AGEcoredns-65dbdb44db-8bxlr               1/1     Running   33         17dkube-ovn-cni-4v4xb                     1/1     Running   0          18dkube-ovn-cni-kvgrj                     1/1     Running   0          18dkube-ovn-cni-nj2pr                     1/1     Running   0          18dkube-ovn-cni-xv476                     1/1     Running   0          18dkube-ovn-controller-7f6db69b48-6c7w8   1/1     Running   0          18dkube-ovn-controller-7f6db69b48-82kjt   1/1     Running   0          18dkube-ovn-controller-7f6db69b48-mhkfc   1/1     Running   0          18dkube-ovn-pinger-n2rn4                  1/1     Running   0          18dkube-ovn-pinger-s4hrz                  1/1     Running   0          18dkube-ovn-pinger-tccz5                  1/1     Running   0          18dkube-ovn-pinger-x2tqq                  1/1     Running   0          18dovn-central-775c4ff46d-4nqjw           1/1     Running   1          18dovn-central-775c4ff46d-822v2           1/1     Running   0          18dovn-central-775c4ff46d-txkn8           1/1     Running   0          18dovs-ovn-mbpv2                          1/1     Running   0          18dovs-ovn-r9mvc                          1/1     Running   0          18dovs-ovn-wkxld                          1/1     Running   0          18dovs-ovn-z89hw                          1/1     Running   0          18d

Invalid value: 0x0: must be specified for an update

https://www.codeleading.com/article/27252474726/

https://github.com/kubevirt/client-pythonhttps://github.com/kubevirt/client-go

创建虚拟机( create_namespaced_virtual_machine)  注意：body是json格式，官方sdk的example有误删除虚拟机( delete_namespaced_virtual_machine)展示某个namespace下的vm资源( list_namespaced_virtual_machine)展示某个namespace下的vmi资源( list_namespaced_virtual_machine_instance)展示所有namespace下的vm资源( list_virtual_machine_for_all_namespaces)展示所有namespace下的vmi资源( list_virtual_machine_instance_for_all_namespaces)获取某个namespace某个name下的vm资源( read_namespaced_virtual_machine)获取某个namespace某个name下的vmi资源( read_namespaced_virtual_machine_instance) 注意：在获取vm资源时无法获取到node_name,uuid,ip的数据，获取vmi资源时无法获取到disk以及image_url的数据，笔者认为vm是虚拟机资源，vmi是虚拟机实例资源，只有在running状态下的vm才是vmi，由于k8s中ip是动态分配的，因此才会出现node_name,uuid,ip数据在vm中获取不到启动虚拟机( v1alpha3_start)停止虚拟机( v1alpha3_stop)重启虚拟机( v1alpha3_restart) 注意: 重启虚拟机只能在虚拟机状态是running是才能调用，否则会失败修改虚拟机名称( v1alpha3_rename)替换虚拟机的配置文件( replace_namespaced_virtual_machine_instance)

def v1alpha3_rename_with_http_info(self, name, newName, namespace, **kwargs):   """   Rename a stopped VirtualMachine object.   This method makes a synchronous HTTP request by default. To make an   asynchronous HTTP request, please define a `callback` function   to be invoked when receiving the response.   >>> def callback_function(response):   >>>     pprint(response)   >>>   >>> thread = api.v1alpha3_rename_with_http_info(name, namespace, newName, callback=callback_function)   :param callback function: The callback function       for asynchronous request. (optional)   :param str name: Name of the resource (required)   :param str namespace: Object name and auth scope, such as for teams and projects (required)   :param str newName: NewName of the resource (required)   :return: str            If the method is called asynchronously,            returns the request thread.   """   all_params = ['name', 'namespace', 'newName']   all_params.append('callback')   all_params.append('_return_http_data_only')   all_params.append('_preload_content')   all_params.append('_request_timeout')   params = locals()   for key, val in iteritems(params['kwargs']):       if key not in all_params:           raise TypeError(               "Got an unexpected keyword argument '%s'"               " to method v1alpha3_rename" % key           )       params[key] = val   del params['kwargs']   # verify the required parameter 'name' is set   if ('name' not in params) or (params['name'] is None):       raise ValueError("Missing the required parameter `name` when calling `v1alpha3_rename`")   # verify the required parameter 'namespace' is set   if ('namespace' not in params) or (params['namespace'] is None):       raise ValueError("Missing the required parameter `namespace` when calling `v1alpha3_rename`")   collection_formats = {}   path_params = {}   # if 'name' in params:   #     path_params['name'] = params['name']   # if 'namespace' in params:   #     path_params['namespace'] = params['namespace']   query_params = []   header_params = {}   form_params = []   local_var_files = {}   body_params = {"newName": params["newName"]}   # Authentication setting   auth_settings = []   api_route = "/apis/subresources.kubevirt.io/v1alpha3/namespaces/{namespace}/virtualmachines/{name}/rename".format(namespace=params["namespace"], name=params["name"])   return self.api_client.call_api(api_route, 'PUT',                                   path_params,                                   query_params,                                   header_params,                                   body=body_params,                                   post_params=form_params,                                   files=local_var_files,                                   response_type='str',                                   auth_settings=auth_settings,                                   callback=params.get('callback'),                                   _return_http_data_only=params.get('_return_http_data_only'),                                   _preload_content=params.get('_preload_content', True),                                   _request_timeout=params.get('_request_timeout'),                                   collection_formats=collection_formats)