Versions Compared

Version Old Version 4 New Version 5
Changes made by

Brett Champion

Brett Champion

Saved on

Key

  • This line was added.
  • This line was removed.
  • Formatting was changed.

...

These settings control the overall deployment environment for your Kubernetes cluster. If you would like a sample values.yaml to start with, you can find one at scopearctl/docs/sample/scopear_k8s_eks_sample.yaml.

Code Block
spec:
metadata:
  name: <clustername>
  namespace: scopear
spec:
  replicas: 1
  logLevel: info
  subdomainSuffix
  hostnames:
    <your primary domain>
    <any secondary domains (optional)>

...

Code Block
cms:
    eksRoleArn: <eks role arn>
    api:
      replicas: 1
      resources: 
        limits:
          cpu: "4"
          memory: 4Gi
        requests:
          cpu: "1"
          memory: 4Gi
    sidekiq:
      replicas: 1
      resources:
        limits:
          cpu: 100m
          memory: 1Gi
        requests:
          cpu: 100m
          memory: 1Gi
    config:
      mysql: 
        host: <mysql host>
        database: <db name>
        username: <username>
        use_iam_auth: false
        aws_rds_region: <region>
      redis: 
        host: <redis host>
        port: 6379
      smtp: # SMTP configuration for email
        enabled: true
        server_host: <smtp host>
        server_port: 537
        sender_address: <e.g. noreply@example.com>
        username: <username>
        smtp_authentication: plain
        smtp_openssl_verify_mode: none
        smtp_enable_starttls_auto: true
        email_format: all
      storage: # Storage configuration
        file_storage: <minio/s3/local>

      admin:
        company_name: Scope AR
        email_address: support@companyA.com
        first_name: test
        last_name: user
        phone_number: 1234-1234-1234
      mysql: 
        host: <mysql host>
        database: <db name>
       storage_size: <storage size>
        volume_name: <volume name>
        volume_mode: <volume mode>
        storage_class_name: <storage class name> port: <port>
        username: <username>
        use_iam_auth: false
        aws_rds_region: <region>
      redis: 
        host: <redis host>
        port: 6379
      smtp: # SMTP configuration for email
        enabled: true
        server_host: <smtp host>
        server_port: 537
        sender_address: <e.g. noreply@example.com>
        username: <username>
        smtp_authentication: plain
        smtp_openssl_verify_mode: none
        smtp_enable_starttls_auto: true
        email_format: all
      storage: # Storage configuration
        file_storage: <minio/s3/local>
        storage_ssl_verify: <true/false>
        default_bucket: <s3 storage bucket with role permissions>
        default_region: <region>

...


      zeus:
        enabled: <true/false>
        external_host: <hostname>
        external_port: <port>

  • cms.eksRoleArn: AWS Role ARN withS3 GetObject, PutObject, and ListBucket permissions for the CMS S3 bucket, as defined in storage.default_bucket. 

Info

Note: If using IAM Auth for RDS or DocumentDB/MongoDB Atlas, IAM policies for connecting to these services will also need to be added. See sections 1.2.1, 1.2.2, and 1.2.3 for more information.

  • cms.config.mysqladmin.host: MySQL hostname. If using inemail_address: The email address of the super user admin. This email address will be the first user seeded in the CMS. Please contact Scope AR for default deployment password.

  • cms.config.mysql.host: MySQL hostname. If using in-cluster MySQL, this can be ignored

  • cms.config.mysql.use_iam_auth: Boolean for using IAM auth for connection to AWS RDS. Defaults to false

  • cms.config.aws_rds_region: AWS region where RDS instance is located. It can be ignored if using in-cluster MySQL

  • cms.config.redis.hostname: Redis hostname. If using in-cluster Redis, this can be ignored

  • cms.config.smtp.smtp_authentication: Specify the authentication type here (plain/login/cram_md5) - default: plain

  • cms.config.smtp.smtp_openssl_verify_mode: If using TLS, set how OpenSSL checks the certificate (none/peer/client_once/fail_if_no_peer_cert). Defaults to 'none'

  • cms.config.smtp.smtp_enable_starttls_auto: Defaults to ‘true’ for SMTP using TLS

  • cms.config.storage.file_storage: Can be one of ‘s3/minio/local’. It is recommended that ‘minio’ be used for local storage. This allows Worklink Create and CMS to use the same storage backend. If set to local

  • cms.config.storage.storage_ssl_verify: If using AWS services or creating trusted SSL certificates for Minio, set to true. If using self-signed certificates for Minio, set to false.

  • cms.config.storage.default_bucket: Name of S3/Minio bucket to use for content storage.

  • cms.config.storage.default_region: If using S3, the default region where S3 bucket is located.

  • cms.config.zeus.enabled: If using remote assist feature, enable zeus.

C2 Service Configuration

This section configures the C2 service, including API, client, project, packager, and model converter settings.

...

Code Block
zeus:
  enabled: true
  annotations:
    service.beta.kubernetes.io/aws-load-balancer-name: scopear-zeus-k8s-prod
    service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags: "eks:cluster-name=k8s-example"
    service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp
    service.beta.kubernetes.io/aws-load-balancer-healthcheck-healthy-threshold: '2'
    service.beta.kubernetes.io/aws-load-balancer-healthcheck-interval: '10'
    service.beta.kubernetes.io/aws-load-balancer-healthcheck-unhealthy-threshold: '2'
    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
    service.beta.kubernetes.io/aws-load-balancer-target-group-attributes: preserve_client_ip.enabled=true
    service.beta.kubernetes.io/aws-load-balancer-type: external
    external-dns.alpha.kubernetes.io/hostname: zeus.examplecompanyA.com 
  config:
    redis:
      host: <redis host>
      port: 6379 # Defaults to 6379
      database: 10 # Defaults to 10
      use_auth: false

...

Code Block
iot:
  enabled: false
  annotations:
    service.beta.kubernetes.io/aws-load-balancer-name: scopear-mqtt-k8s-prod
    service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags: "eks:cluster-name=k8s-example"
    service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp
    service.beta.kubernetes.io/aws-load-balancer-healthcheck-healthy-threshold: '2'
    service.beta.kubernetes.io/aws-load-balancer-healthcheck-interval: '10'
    service.beta.kubernetes.io/aws-load-balancer-healthcheck-unhealthy-threshold: '2'
    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
    service.beta.kubernetes.io/aws-load-balancer-target-group-attributes: preserve_client_ip.enabled=true
    service.beta.kubernetes.io/aws-load-balancer-type: external
    external-dns.alpha.kubernetes.io/hostname: mqtt.example.com 
  port: 9000 # defaults to 9000
  ssl_enabled: true

  • iot.annotations: IoT allows you to add annotations if you are deploying to a cloud provider and would like to use annotations to provision the NLB. The annotations provided above are only an example. Please use annotations that fit your cloud provider. If you are provisioning a load balancer outside annotations, please ignore this section.

...

In-Cluster Datastores

If your organization does not have the ability to cannot leverage cloud services for redisRedis/s3/mysqlMySQL/mongodbMongoDB, you will need to configure the datastores within the cluster.data stores within the cluster. Kubernetes supports dynamic provisioning of storage volumes, which means that when you create a PersistentVolumeClaim (PVC), Kubernetes can automatically provision the underlying storage resource (e.g., a disk in a public cloud, an NFS share, a Ceph RBD volume, etc.). However, if you cannot leverage dynamic provisioning and need to create the volumes manually beforehand, we support specifying volume names in your volume claim templates.

Dynamic Provisioning

Below is an example of deploying in-cluster data stores, leveraging cloud-based storage provisioners. This method is typical when the admin wants to deploy to EKS, AKS, GKE, dynamically, utilizing the default storage controller/storage class name configured in the cluster. If leveraging external datastores like S3, RDS, DocumentDB, please set enabled: false

Code Block
  minio:
    enabled: true
    storage_size: <storage size>
    storage_class_name: <class name>
    replicas: 1
    resources: {}
  mongodb:
    enabled: true
    storage_size: <storage size>
    storage_class_name: <class name>
    replicas: 1
    resources: {}
  mysql:
    enabled: true
    storage_size: <storage size>
    storage_class_name: <class name>
    replicas: 1
    resources: {}
  redis:
    enabled: true
    storage_size: <storage size>
    storage_class_name: <class name>
    replicas: 1
    resources: {}

Additionally, if S3 is not a storage option, and you are migrating from a previous installation where local storage was used for CMS, you will need to specify the following:

Code Block
cms:
  config:      
      storage: # Storage configuration
        file_storage: local
        storage_size: <storage size>
        volume_name: <volume name>
        volume_mode: <volume mode>
        storage_class_name: <storage class name>
        storage_ssl_verify: <true/false>
        default_bucket: <s3 storage bucket with role permissions>
        default_region: <region>

  • cms.config.storage.storage_size: Size of the disk to be mounted in CMS pod.

  • cms.config.storage.volume_name: If pre-built disks are desired, specify the volume name.

  • cms.config.storage.volume_mode: Volume mode of the disk. If not specified, this will use the default volume mode of storage provisioned. (e.g. Block, Filesystem, etc)

  • cms.config.storage_class_name: Storage class name to utilize. If not specified, will use the default storage class name of the provider. (e.g. gp2, nfs-csi, standard, etc)

Non-Dynamic Provisioning

Code Block
  minio:
    enabled: <true/false>
    storage_size: <storage size>
    storage_class_name: <class name>
    volume_mode: <volume mode>
    volume_name: <volume name>
    resources: {}
  mongodb:
    enabled: <true/false>
    storage_size: <storage size>
    storage_class_name: <class name>
    volume_mode: <volume mode>
    volume_name: <volume name>
    replicas: 1
    resources: {}
  mysql:
    enabled: <true/false>
    storage_size: <storage size>
    storage_class_name: <class name>
    volume_mode: <volume mode>
    volume_name: <volume name>
    replicas: 1
    resources: {}
  redis:
    enabled: <true/false>
    storage_size: <storage size>
    storage_class_name: <class name>
    volume_mode: <volume mode>
    volume_name: <volume name>
    replicas: 1
    resources: {}

  • storage_size: Even when specifying volume_name of a pre-configured disk, you must specify the desired storage size, making sure it is >= the actual size of the pre-built volume.

  • storage_class_name: Storage class name to utilize. If not specified, will use the default storage class name of the provider. (e.g. gp2, nfs-csi, standard, etc)

  • volume_mode: Volume mode of the disk. If not specified, this will use the default volume mode of storage provisioned. (e.g. Block, Filesystem, etc)

  • volume_name:If pre-built disks are desired, specify the volume name.

Full Example - AWS EKS

Below is an example values.yaml configuration for an AWS EKS deployment, utilizing AWS Services for the datastores with Pixyz and IoT enabled:

Code Block
apiVersion: k8s.scopear.com/v1
kind: ScopeAR
metadata:
  name: scopear-dev1
  namespace: scopear
spec:
  channel: stable
  env: intranet
  replicas: 3
  logLevel: debug
  hostnames:
    - companyA.com

  cluster:
    provider: aws
    type: eks
    region: us-west-2

  cms:
    eksRoleArn: arn:aws:iam::939303481:role/eks-s3.scopear-dev1
    api:
      replicas: 3
      resources:
        limits:
          cpu: "4"
          memory: 4Gi
        requests:
          cpu: "1"
          memory: 4Gi
    sidekiq:
      replicas: 2
      resources:
        requests:
          cpu: "1"
          memory: 4Gi
        limits:
          cpu: "4"
          memory: 4Gi
    config:
      admin:
        company_name: Scope AR
        email_address: support@scopear.com
        first_name: ScopeAR
        last_name: Support
        phone_number: 780-555-1234
      mysql:
        database: scope_cms_production
        host: scopear-test-cms-2022-02-05-cluster.cluster-cuasd1uy.us-west-2.rds.amazonaws.com
        port: 3306
        username: scope_production
        use_iam_auth: true
      redis:
        host: redis
        port: 6379
      smtp:
        enabled: true
        ehlo_domain: mail.test.com
        email_format: all
        sender_address: no-reply@scopear.com
        server_host: smtp.test.org
        server_port: 587
        smtp_authentication: plain
        username: postmaster@mail.test.com
      storage:
        file_storage: s3
        default_bucket: test-content-us-west-2
        default_region: us-west-2
      zeus:
        enabled: true
        external_host: zeus-dev1.companyA.com
        external_port: 24000

  c2:
    enabled: true
    eksRoleArn: arn:aws:iam::9392939322481:role/eks-s3.scopear-dev1
    api:
      replicas: 3
      resources:
        limits:
          cpu: 500m
          memory: 3000Mi
        requests:
          cpu: 200m
          memory: 3000Mi
    client:
      replicas: 3
      resources:
        limits:
          cpu: 100m
          memory: 100Mi
        requests:
          cpu: 100m
          memory: 100Mi
    model_converter:
      replicas: 2
      resources:
        limits:
          cpu: 4000m
          memory: 15G
        requests:
          cpu: 2000m
          memory: 12G
    project_packager:
      replicas: 3
      resources:
        limits:
          cpu: 2
          memory: 5000Mi
        requests:
          cpu: 0.5
          memory: 2000Mi
    config:
      cad_pipeline:
        query_time: 10000
        processing_mode: pixyz_local
        flex_host: flexlm.companyA.com
        flex_port: 27000
      mongodb:
        host: scopear-eks-dev-1-c2-docdb.cluster-cu4c1sdfuy.us-west-2.docdb.amazonaws.com
        username: mongodb
        port: 27017
        database: "test?ssl=true&retryWrites=false&tlsInsecure=true"
      storage:
        storage_provider: aws
        bucket_name: companyA-worklink-create-2-dev
        aws_bucket_region: us-west-2

  dt:
    enabled: true
    replicas: 1

  iot:
    enabled: true
    port: 9000
    annotations:
      service.beta.kubernetes.io/aws-load-balancer-name: scopear-mqtt-k8s-prod
      service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags: "eks:cluster-name=k8s-example"
      service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp
      service.beta.kubernetes.io/aws-load-balancer-healthcheck-healthy-threshold: '2'
      service.beta.kubernetes.io/aws-load-balancer-healthcheck-interval: '10'
      service.beta.kubernetes.io/aws-load-balancer-healthcheck-unhealthy-threshold: '2'
      service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
      service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
      service.beta.kubernetes.io/aws-load-balancer-target-group-attributes: preserve_client_ip.enabled=true
      service.beta.kubernetes.io/aws-load-balancer-type: external
      external-dns.alpha.kubernetes.io/hostname: mqtt.example.com 
    config:
      mongodb:
        host: scopear-eks-dev-1-c2-docdb.cluster-cu4c1yasdf1uy.us-west-2.docdb.amazonaws.com
        port: 27017
        database: "iot?ssl=true&retryWrites=false&tlsInsecure=true"
        username: mongodb

  zeus:
    enabled: true
    external_port: 24000
    annotations:
      service.beta.kubernetes.io/aws-load-balancer-name: scopear-zeus-k8s-prod
      service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags: "eks:cluster-name=k8s-example"
      service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp
      service.beta.kubernetes.io/aws-load-balancer-healthcheck-healthy-threshold: '2'
      service.beta.kubernetes.io/aws-load-balancer-healthcheck-interval: '10'
      service.beta.kubernetes.io/aws-load-balancer-healthcheck-unhealthy-threshold: '2'
      service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
      service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
      service.beta.kubernetes.io/aws-load-balancer-target-group-attributes: preserve_client_ip.enabled=true
      service.beta.kubernetes.io/aws-load-balancer-type: external
      external-dns.alpha.kubernetes.io/hostname: zeus-dev1.companyA.com
    config:
      redis:
        host: redis
        port: 6379
        database: 8
        use_auth: false

  minio:
    enabled: false
  mongodb:
    enabled: false
  mysql:
    enabled: false
  redis:
    enabled: false

Full Example - In-Cluster Data Stores

Below is an example of deploying Scope AR into an on-prem Openshift cluster with in-cluster data stores (minio/mysql/mongodb/redis):

Code Block
apiVersion: k8s.scopear.com/v1
kind: ScopeAR
metadata:
  name: scopear-dev1
  namespace: scopear
spec:
  channel: stable
  env: intranet
  replicas: 3
  logLevel: debug
  hostnames:
    - companyA.com

  cluster:
    provider: aws
    type: eks
    region: us-west-2

  cms:
    api:
      replicas: 3
      resources:
        limits:
          cpu: "4"
          memory: 4Gi
        requests:
          cpu: "1"
          memory: 4Gi
    sidekiq:
      replicas: 2
      resources:
        requests:
          cpu: "1"
          memory: 4Gi
        limits:
          cpu: "4"
          memory: 4Gi
    config:
      admin:
        company_name: Scope AR
        email_address: support@scopear.com
        first_name: ScopeAR
        last_name: Support
        phone_number: 780-555-1234
      mysql:
        database: scope_cms_production
        host: mysql # k8s mysql service name
        port: 3306
        username: scope_dev
      redis:
        host: redis
        port: 6379
      smtp:
        enabled: true
        ehlo_domain: mail.test.com
        email_format: all
        sender_address: no-reply@test.com
        server_host: smtp.test.org
        server_port: 587
        smtp_authentication: plain
        username: postmaster@mail.test.com
      storage:
        file_storage: minio
        default_bucket: test-content-us-west-2
      zeus:
        enabled: true
        external_host: zeus-dev1.companyA.com
        external_port: 24000

  c2:
    enabled: true
    api:
      replicas: 3
      resources:
        limits:
          cpu: 500m
          memory: 3000Mi
        requests:
          cpu: 200m
          memory: 3000Mi
    client:
      replicas: 3
      resources:
        limits:
          cpu: 100m
          memory: 100Mi
        requests:
          cpu: 100m
          memory: 100Mi
    model_converter:
      replicas: 2
      resources:
        limits:
          cpu: 4000m
          memory: 15G
        requests:
          cpu: 2000m
          memory: 12G
    project_packager:
      replicas: 3
      resources:
        limits:
          cpu: 2
          memory: 5000Mi
        requests:
          cpu: 0.5
          memory: 2000Mi
    config:
      cad_pipeline:
        query_time: 10000
        processing_mode: pixyz_local
        flex_host: flexlm.companyA.com
        flex_port: 27000
      mongodb:
        host: mongodb.qa1-scopear.svc.cluster.local # Need full service fqdn
        username: mongodb
        port: 27017
        database: "test?ssl=true&retryWrites=false&tlsInsecure=true"
      storage:
        storage_provider: minio
        minio_server_address: minio-dev1.companyA.com
        minio_server_port: 443
        minio_server_use_ssl: true
        bucket_name: create

  dt:
    enabled: true
    replicas: 1

  iot:
    enabled: true
    port: 9000
    config:
      mongodb:
        host: mongodb
        port: 27017
        database: "iot"
        username: mongodb

  zeus:
    enabled: true
    external_port: 24000
    config:
      redis:
        host: redis
        port: 6379
        database: 8
        use_auth: false

  minio:
    enabled: true
    storage_size: 50G
    storage_class_name: thin
    volume_mode: Filesystem
    volume_name: minio-pv1
    resources: {}
  mongodb:
    enabled: true
    storage_size: 50G
    storage_class_name: thin
    volume_mode: Filesystem
    volume_name: mongodb-pv1
    replicas: 1
    resources: {}
  mysql:
    enabled: true
    storage_size: 50G
    storage_class_name: Filesystem
    volume_mode: thin
    volume_name: mysql-pv1
    replicas: 1
    resources: {}
  redis:
    enabled: true
    storage_size: 50G
    storage_class_name: thin
    volume_mode: thin
    volume_name: redis-pv1
    replicas: 1
    resources: {}