我已经为此连续工作了将近5天,无法使它正常工作。根据AWS文档,我应该*能够将EFS卷安装到部署到kubernetes(EKS)中的Fargate节点的Pod中。
我正在通过terraform 100%完成一切。在这一点上,我迷路了,而我所阅读的可怕文档几乎使我的眼睛流血。任何人都可以给我的有关使此工作正常进行的指导将是惊人的!
到目前为止,这是我所做的:

  • 设置EKS CSI驱动程序,存储类和角色绑定(bind)(不确定我为什么需要这些角色绑定(bind)tbh)
  • resource "kubernetes_csi_driver" "efs" {
      metadata {
        name = "efs.csi.aws.com"
      }
    
      spec {
        attach_required        = false
        volume_lifecycle_modes = [
          "Persistent"
        ]
      }
    }
    
    resource "kubernetes_storage_class" "efs" {
      metadata {
        name = "efs-sc"
      }
      storage_provisioner = kubernetes_csi_driver.efs.metadata[0].name
      reclaim_policy      = "Retain"
    }
    
    resource "kubernetes_cluster_role_binding" "efs_pre" {
      metadata {
        name = "efs_role_pre"
      }
      role_ref {
        api_group = "rbac.authorization.k8s.io"
        kind      = "ClusterRole"
        name      = "cluster-admin"
      }
      subject {
        kind      = "ServiceAccount"
        name      = "default"
        namespace = "pre"
      }
    }
    
    resource "kubernetes_cluster_role_binding" "efs_live" {
      metadata {
        name = "efs_role_live"
      }
      role_ref {
        api_group = "rbac.authorization.k8s.io"
        kind      = "ClusterRole"
        name      = "cluster-admin"
      }
      subject {
        kind      = "ServiceAccount"
        name      = "default"
        namespace = "live"
      }
    }
    
  • 设置带有策略和安全组
  • 的EFS卷
    module "vpc" {
      source    = "../../read_only_data/vpc"
      stackname = var.vpc_stackname
    }
    resource "aws_efs_file_system" "efs_data" {
        creation_token = "xva-${var.environment}-pv-efsdata-${var.side}"
    
        # encrypted   = true
        # kms_key_id  = ""
    
        performance_mode = "generalPurpose" #maxIO
        throughput_mode  = "bursting"
    
        lifecycle_policy {
            transition_to_ia = "AFTER_30_DAYS"
        }
    }
    
    data "aws_efs_file_system" "efs_data" {
      file_system_id = aws_efs_file_system.efs_data.id
    }
    
    resource "aws_efs_access_point" "efs_data" {
      file_system_id = aws_efs_file_system.efs_data.id
    }
    
    /* Policy that does the following:
    - Prevent root access by default
    - Enforce read-only access by default
    - Enforce in-transit encryption for all clients
    */
    resource "aws_efs_file_system_policy" "efs_data" {
      file_system_id = aws_efs_file_system.efs_data.id
    
      policy = jsonencode({
        "Version": "2012-10-17",
        "Statement": [
            {
                "Effect": "Allow",
                "Principal": {
                    "AWS": "*"
                },
                "Action": "elasticfilesystem:ClientMount",
                "Resource": aws_efs_file_system.efs_data.arn
            },
            {
                "Effect": "Deny",
                "Principal": {
                    "AWS": "*"
                },
                "Action": "*",
                "Resource": aws_efs_file_system.efs_data.arn,
                "Condition": {
                    "Bool": {
                        "aws:SecureTransport": "false"
                    }
                }
            }
        ]
      })
    }
    
    # Security Groups for this volume
    resource "aws_security_group" "allow_eks_cluster" {
      name        = "xva-${var.environment}-efsdata-${var.side}"
      description = "This will allow the cluster ${data.terraform_remote_state.cluster.outputs.eks_cluster_name} to access this volume and use it."
      vpc_id      = module.vpc.vpc_id
    
      ingress {
        description = "NFS For EKS Cluster ${data.terraform_remote_state.cluster.outputs.eks_cluster_name}"
        from_port   = 2049
        to_port     = 2049
        protocol    = "tcp"
        security_groups = [
          data.terraform_remote_state.cluster.outputs.eks_cluster_sg_id
        ]
      }
    
      egress {
        from_port   = 0
        to_port     = 0
        protocol    = "-1"
        cidr_blocks = ["0.0.0.0/0"]
      }
    
      tags = {
        Name = "allow_tls"
      }
    }
    
    # Mount to the subnets that will be using this efs volume
    # Also attach sg's to restrict access to this volume
    resource "aws_efs_mount_target" "efs_data-app01" {
      file_system_id = aws_efs_file_system.efs_data.id
      subnet_id      = module.vpc.private_app_subnet_01
    
      security_groups = [
        aws_security_group.allow_eks_cluster.id
      ]
    }
    
    resource "aws_efs_mount_target" "efs_data-app02" {
      file_system_id = aws_efs_file_system.efs_data.id
      subnet_id      = module.vpc.private_app_subnet_02
    
      security_groups = [
        aws_security_group.allow_eks_cluster.id
      ]
    }
    
  • 参考kubernetes中的EFS卷创建一个持久卷
  • data "terraform_remote_state" "csi" {
      backend = "s3"
      config = {
        bucket  = "xva-${var.account_type}-terraform-${var.region_code}"
        key     = "${var.environment}/efs/driver/terraform.tfstate"
        region  = var.region
        profile = var.profile
      }
    }
    resource "kubernetes_persistent_volume" "efs_data" {
      metadata {
        name = "pv-efsdata"
    
        labels = {
            app = "example"
        }
      }
    
      spec {
        access_modes = ["ReadOnlyMany"]
    
        capacity = {
          storage = "25Gi"
        }
    
        volume_mode                      = "Filesystem"
        persistent_volume_reclaim_policy = "Retain"
        storage_class_name               = data.terraform_remote_state.csi.outputs.storage_name
    
        persistent_volume_source {
          csi {
            driver        = data.terraform_remote_state.csi.outputs.csi_name
            volume_handle = aws_efs_file_system.efs_data.id
            read_only    = true
          }
        }
      }
    }
    
  • 然后创建一个部署以通过挂载EFS卷
  • 的Pod进行部署
    data "terraform_remote_state" "efs_data_volume" {
      backend = "s3"
      config = {
        bucket  = "xva-${var.account_type}-terraform-${var.region_code}"
        key     = "${var.environment}/efs/volume/terraform.tfstate"
        region  = var.region
        profile = var.profile
      }
    }
    resource "kubernetes_persistent_volume_claim" "efs_data" {
      metadata {
        name      = "pv-efsdata-claim-${var.side}"
        namespace = var.side
      }
    
      spec {
        access_modes       = ["ReadOnlyMany"]
        storage_class_name =  data.terraform_remote_state.csi.outputs.storage_name
        resources {
          requests = {
            storage = "25Gi"
          }
        }
        volume_name = data.terraform_remote_state.efs_data_volume.outputs.volume_name
      }
    }
    
    resource "kubernetes_deployment" "example" {
      timeouts {
        create = "3m"
        update = "4m"
        delete = "2m"
      }
    
      metadata {
        name      = "deployment-example"
        namespace = var.side
    
        labels = {
          app      = "example"
          platform = "fargate"
          subnet   = "app"
        }
      }
    
      spec {
        replicas = 1
    
        selector {
          match_labels = {
            app = "example"
          }
        }
    
        template {
          metadata {
            labels = {
              app      = "example"
              platform = "fargate"
              subnet   = "app"
            }
          }
    
          spec {
            volume {
              name = "efs-data-volume"
    
              persistent_volume_claim {
                claim_name = kubernetes_persistent_volume_claim.efs_data.metadata[0].name
                read_only  = true
              }
            }
    
            container {
              image = "${var.nexus_docker_endpoint}/example:${var.docker_tag}"
              name  = "example"
    
              env {
                name  = "environment"
                value = var.environment
              }
              env {
                name = "dockertag"
                value = var.docker_tag
              }
    
              volume_mount {
                name = "efs-data-volume"
                read_only = true
                mount_path = "/appconf/"
              }
    
              # liveness_probe {
              #   http_get {
              #     path = "/health"
              #     port = 443
              #   }
    
              #   initial_delay_seconds = 3
              #   period_seconds        = 3
              # }
    
              port {
                container_port = 443
              }
            }
          }
        }
      }
    }
    
    它可以看到kuberenetes中的持久性卷,我可以看到它已被声明,甚至可以看到它试图将该卷安装到pod日志中。但是,在描述广告连播时,我不可避免地总是看到以下错误:
    Volumes:
      efs-data-volume:
        Type:        PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
        ClaimName:   pv-efsdata-claim-pre
        ReadOnly:    true
    ...
    ...
    Events:
      Type     Reason       Age                  From                                                       Message
      ----     ------       ----                 ----                                                       -------
      Warning  FailedMount  11m (x629 over 23h)  kubelet, <redacted-fargate-endpoint>  Unable to attach or mount volumes: unmounted volumes=[efs-data-volume], unattached volumes=[efs-data-volume]: timed out waiting for the condition
      Warning  FailedMount  47s (x714 over 23h)  kubelet, <redacted-fargate-endpoint>  MountVolume.SetUp failed for volume "pv-efsdata" : kubernetes.io/csi: mounter.SetupAt failed: rpc error: code = InvalidArgument desc = Volume capability not supported
    

    最佳答案

    我终于做到了。我已经成功将EFS卷安装到Fargate Pod(近6天后)!我可以从这个封闭的github问题中找到所需的方向:https://github.com/aws/containers-roadmap/issues/826
    最终导致我正在使用此模块构建我的eks集群:https://registry.terraform.io/modules/cloudposse/eks-cluster/aws/0.29.0?tab=outputs
    如果使用输出“security_group_id”,则输出“其他安全组”。以我的经验,这绝对对aws毫无好处。不知道为什么当您无能为力时它甚至存在。我需要使用的安全组是“群集安全组”。因此,我在EFS卷安全组安装点和BAM的端口2049入口规则上添加了“群集安全组”的ID!我将EFS卷成功安装到已部署的Pod。
    另一个重要的更改是我将持久性卷类型更改为ReadWriteMany,因为fargate显然不支持ReadOnlyMany。

    关于kubernetes - Terraform AWS EKS-无法将EFS卷挂载到Fargate Pod,我们在Stack Overflow上找到一个类似的问题:https://stackoverflow.com/questions/64250161/

    10-11 06:48