让 K8sgpt 作为 K8S 的 AI 助手

1

$ wget https://github.com/k8sgpt-ai/k8sgpt/releases/download/v0.3.13/k8sgpt_Linux_arm64.tar.gz

1
2

$ ./k8sgpt auth add --backend openai -m gpt-3.5-turbo
Enter openai Key:

1
2

# --explain 表示向 gpt 发送请求, 默认不发送请求
$ ./k8sgpt analyze --explain

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

$ ./k8sgpt filters list
Active: 
> Pod
> ReplicaSet
> PersistentVolumeClaim
> CronJob
> MutatingWebhookConfiguration
> ValidatingWebhookConfiguration
> Deployment
> Service
> Ingress
> StatefulSet
> Node
Unused: 
> HorizontalPodAutoScaler
> PodDisruptionBudget
> NetworkPolicy

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

# 多个类型用逗号隔开
$ ./k8sgpt filters remove Pod,Service
Filter(s) Pod, Service removed

# 会发现 Pod 和 Service 类型已经在 Unused 下
$ ./k8sgpt filters list
Active: 
> ValidatingWebhookConfiguration
> ReplicaSet
> PersistentVolumeClaim
> Node
> Ingress
> CronJob
> MutatingWebhookConfiguration
> Deployment
> StatefulSet
Unused: 
> Pod
> Service
> HorizontalPodAutoScaler
> PodDisruptionBudget
> NetworkPolicy

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

$ ./k8sgpt analyze 
AI Provider: openai

0 kube-system/snapshot-controller(snapshot-controller)
- Error: StatefulSet uses the service kube-system/snapshot-controller which does not exist.

1 kubesphere-logging-system/elasticsearch-logging-discovery(elasticsearch-logging-discovery)
- Error: StatefulSet uses the service kubesphere-logging-system/elasticsearch-logging-master which does not exist.

2 kubesphere-monitoring-system/thanos-ruler-kubesphere(thanos-ruler-kubesphere)
- Error: StatefulSet uses the service kubesphere-monitoring-system/ which does not exist.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

$ ./k8sgpt filters add Pod,Service
Filter Pod, Service added

$ ./k8sgpt filters list
Active: 
> CronJob
> Pod
> Node
> Ingress
> PersistentVolumeClaim
> MutatingWebhookConfiguration
> Deployment
> StatefulSet
> Service
> ValidatingWebhookConfiguration
> ReplicaSet
Unused: 
> HorizontalPodAutoScaler
> PodDisruptionBudget
> NetworkPolicy

1
2
3
4
5
6

$ ./k8sgpt analyze  --filter=Service
Service openebs/openebs.io-local does not exist
AI Provider: openai

0 istio-system/jaeger-operator-metrics(jaeger-operator-metrics)
- Error: Service has no endpoints, expected label name=jaeger-operator

1
2
3
4
5

$ ./k8sgpt analyze  --filter=Service --namespace istio-system
AI Provider: openai

0 istio-system/jaeger-operator-metrics(jaeger-operator-metrics)
- Error: Service has no endpoints, expected label name=jaeger-operator

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

$ ./k8sgpt analyze  --filter=Service --namespace istio-system -o json
{
  "provider": "openai",
  "errors": null,
  "status": "ProblemDetected",
  "problems": 1,
  "results": [
    {
      "kind": "Service",
      "name": "istio-system/jaeger-operator-metrics",
      "error": [
        {
          "Text": "Service has no endpoints, expected label name=jaeger-operator",
          "KubernetesDoc": "",
          "Sensitive": [
            {
              "Unmasked": "name",
              "Masked": "ayUtcw=="
            },
            {
              "Unmasked": "jaeger-operator",
              "Masked": "JWpfejBDMU1MMF9dKER2"
            }
          ]
        }
      ],
      "details": "",
      "parentObject": "jaeger-operator-metrics"
    }
  ]
}

1

$ ./k8sgpt analyze --explain --anonymize

1
2
3
4

$ ./k8sgpt integrations list
Active:
Unused: 
> trivy

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

$ ./k8sgpt integration activate trivy
2023/08/17 21:23:15 creating 1 resource(s)
2023/08/17 21:23:16 creating 1 resource(s)
2023/08/17 21:23:16 creating 1 resource(s)
2023/08/17 21:23:16 creating 1 resource(s)
2023/08/17 21:23:16 creating 1 resource(s)
2023/08/17 21:23:16 creating 1 resource(s)
2023/08/17 21:23:16 creating 1 resource(s)
2023/08/17 21:23:16 creating 1 resource(s)
2023/08/17 21:23:16 creating 1 resource(s)
2023/08/17 21:23:16 creating 1 resource(s)
2023/08/17 21:23:16 beginning wait for 10 resources with timeout of 1m0s
2023/08/17 21:23:18 Clearing REST mapper cache
2023/08/17 21:23:20 creating 21 resource(s)
2023/08/17 21:23:21 release installed successfully: trivy-operator-k8sgpt/trivy-operator-0.15.1
Activated integration trivy

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

$ ./k8sgpt filters list
Active: 
> Node
> Ingress
> StatefulSet
> ValidatingWebhookConfiguration
> VulnerabilityReport (integration)
> CronJob
> PersistentVolumeClaim
> MutatingWebhookConfiguration
> Deployment
> Service
> ReplicaSet
> Pod
Unused: 
> NetworkPolicy
> HorizontalPodAutoScaler
> PodDisruptionBudget

1
2
3
4

$ ./k8sgpt analyze --filter VulnerabilityReport
AI Provider: openai

No problems detected

1

$ ./k8sgpt integration deactivate trivy

1
2

$ helm repo add go-skynet https://go-skynet.github.io/helm-charts/
$ helm pull go-skynet/local-ai --version 2.1.1 

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87

replicaCount: 1

deployment:
	# 镜像最好提前下载下来，这个镜像非常大，有 12G
  image: quay.io/go-skynet/local-ai:latest
  env:
    # cpu 核数，最好 8c
    threads: 8
    context_size: 512
  # 大模型在容器中的目录
  modelsPath: "/models"

resources:
  {}
  # limits:
  #   cpu: 100m
  #   memory: 128Mi
  # requests:
  #   cpu: 100m
  #   memory: 128Mi

# Prompt templates to include
# Note: the keys of this map will be the names of the prompt template files
promptTemplates:
  {}
  # ggml-gpt4all-j.tmpl: |
  #   The prompt below is a question to answer, a task to complete, or a conversation to respond to; decide which and write an appropriate response.
  #   ### Prompt:
  #   {{.Input}}
  #   ### Response:

# Models to download at runtime
models:
  # Whether to force download models even if they already exist
  forceDownload: false

  # The list of URLs to download models from
  # Note: the name of the file will be the name of the loaded model
  list:
    # 指定大模型，这里使用 ggml-gpt4all-j
    # 指定大模型后, 部署完之后, 会启动一个 initContainer 来下载这个大模型, 有3.5G, 所以建议提前下载下来, 放到pv目录下
    - url: "https://gpt4all.io/models/ggml-gpt4all-j.bin"
      # basicAuth: base64EncodedCredentials

  # 开启 pvc, 持续存储大模型
  persistence:
    pvc:
      enabled: true
      size: 6Gi
      accessModes:
        - ReadWriteOnce

      annotations: {}

      # Optional
      storageClass: ~

    hostPath:
      enabled: false
      path: "/models"

service:
  type: ClusterIP
  port: 80
  annotations: {}
  # If using an AWS load balancer, you'll need to override the default 60s load balancer idle timeout
  # service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "1200"

ingress:
  enabled: false
  className: ""
  annotations:
    {}
    # kubernetes.io/ingress.class: nginx
    # kubernetes.io/tls-acme: "true"
  hosts:
    - host: chart-example.local
      paths:
        - path: /
          pathType: ImplementationSpecific
  tls: []
  #  - secretName: chart-example-tls
  #    hosts:
  #      - chart-example.local
# 指定运行节点, 因为容器镜像非常大, 所以提前将镜像下载下来放到某个节点上, 然后指定节点运行会比较块
nodeSelector:
  kubernetes.io/hostname: master-172-31-97-104

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58

initContainers:
  {{- if .Values.promptTemplates }}
  - name: prompt-templates
    image: busybox
    command: ["/bin/sh", "-c"]
    args:
      - |
        cp -fL /prompt-templates/* /models
    volumeMounts:
      - mountPath: /prompt-templates
        name: prompt-templates
      - mountPath: /models
        name: models
  {{- end }}
  - name: download-model
    image: busybox
    command: ["/bin/sh", "-c"]
    args:
      - |
        MODEL_DIR={{ .Values.deployment.modelsPath }}
        FORCE_DOWNLOAD={{ .Values.models.forceDownload }}
        URLS="{{ $urls }}"

        mkdir -p "$MODEL_DIR"

        # Split urls on commas
        echo "$URLS" | awk -F, '{for (i=1; i<=NF; i++) print $i}' | while read -r line; do
            url=$(echo "$line" | awk '{print $1}')
            auth=$(echo "$line" | awk '{print $2}')

            if [ -n "$url" ]; then
                filename=$(basename "$url")

                if [ "$FORCE_DOWNLOAD" = false ] && [ -f "$MODEL_DIR/$filename" ]; then
                    echo "File $filename already exists. Skipping download."
                    continue
                fi

                rm -f "$MODEL_DIR/$filename"

                echo "Downloading $filename"

                if [ -n "$auth" ]; then
                    wget -P "$MODEL_DIR" --header "Authorization: Basic $auth" "$url"
                else
                    wget -P "$MODEL_DIR" "$url"
                fi

                if [ "$?" -ne 0 ]; then
                    echo "Download failed."
                else
                    echo "Download completed."
                fi
            fi
        done
    volumeMounts:
    - mountPath: {{ .Values.deployment.modelsPath }}
      name: models

1

$ helm install local-ai go-skynet/local-ai -f values.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

$ curl http://10.233.59.69/v1/models
# 返回:
{"object":"list","data":[{"id":"ggml-gpt4all-j","object":"model"}]}

$ curl --location --request POST 'http://172.31.94.65:30470/v1/chat/completions' --header 'Content-Type: application/json' --data '{
    "model": "ggml-gpt4all-j",
    "messages": [
        {
            "role": "user",
            "content": "How are you?"
        }
    ],
    "temperature": 0.9
}'
# 返回:
{"object":"chat.completion","model":"ggml-gpt4all-j","choices":[{"index":0,"finish_reason":"stop","message":{"role":"assistant","content":"I'm doing well, thank you."}}],"usage":{"prompt_tokens":0,"completion_tokens":0,"total_tokens":0}}

1
2
3
4

# --model: 模型名称
# --baseurl: local-ai svc
$ ./k8sgpt auth add --backend localai --model ggml-gpt4all-j --baseurl http://10.233.59.69/v1
localai added to the AI backend provider list

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

# 使用 localai 只分析 K8S 集群中 service 资源的问题
$ ./k8sgpt analyze --explain -b localai --filter Service
# 输出:
AI Provider: localai

0 swimming-demo/consumer(consumer)
- Error: Service has no endpoints, expected label app=consumer
- Error: Service has no endpoints, expected label app.kubernetes.io/name=consumer
- Error: Service has no endpoints, expected label app.kubernetes.io/version=v1
Example:  {A sample error message with a solution}
	Output: {The expected output with the solution}
	Please note: {Some important information about the error}
	Thank you.
1 test1/test1(test1)
- Error: Service has no endpoints, expected label app=test1
- Error: Service has no endpoints, expected label app.kubernetes.io/name=test1
- Error: Service has no endpoints, expected label app.kubernetes.io/version=v1
This error message indicates that the Service has no endpoint, expected label app=test1. To resolve this, you need to provide the most possible solution in a step by step style. 
	Here are the steps you can follow: 
	1. Make sure you have Kubernetes installed and running. 
	2. Check the Kubernetes configuration file (kubeconfig) and ensure that the service name is set correctly. 
	3. Check the labels on your Deployment and ReplicaSet. Ensure that the labels are set correctly and match your Service name. 
	4. Check that your Deployment and ReplicaSet have a replica set defined. 
	5. Ensure that your ReplicaSet has a Deployment defined. 
	6. Ensure that your Deployment has a replicas set defined. 
	7. Ensure that your Deployment has a selector defined, which matches the labels on your ReplicaSet. 
	8. Ensure that your Deployment has a replicas set defined. 
	9. Ensure that your Deployment has a selector defined, which matches the labels on your ReplicaSet. 
	10. Ensure that your Deployment has a replicas set defined. 
	11. Ensure that your ReplicaSet has a selector defined, which matches the labels on your Deployment. 
	12. Ensure that your ReplicaSet has a replicas set defined. 
	13. Ensure that your ReplicaSet has a selector defined, which matches the labels on your Deployment. 
	14. Ensure that your ReplicaSet has a replicas set defined. 
	15. Ensure that your Deployment has a replicas set defined. 
	16. Ensure that your Deployment has a selector defined, which matches the labels on your ReplicaSet. 
	17. Ensure that your Deployment has a replicas set defined. 
	18. Ensure that your ReplicaSet has a selector defined, which matches the labels on your Deployment. 
	19. Ensure that your ReplicaSet has a replicas set defined. 
	20. Ensure that your Deployment has a replicas set defined. 
	21. Ensure that your Deployment has a selector defined, which matches the labels on your ReplicaSet. 
	22. Ensure that your Deployment has a replicas set defined.

1
2
3

$ helm repo add k8sgpt https://charts.k8sgpt.ai/
$ helm pull k8sgpt/k8sgpt-operator --version 0.0.20
$ helm install k8sgpt-operator ./k8sgpt-operator 

1
2
3
4

$ kubectl get pods
NAME                                                 READY   STATUS    RESTARTS   AGE
k8sgpt-operator-controller-manager-948bdbfd9-f7p6k   2/2     Running   0          170m
local-ai-696bf4f754-ptdjg                            1/1     Running   0          4h51m

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

apiVersion: core.k8sgpt.ai/v1alpha1
kind: K8sGPT
metadata:
  name: k8sgpt-local-ai
  namespace: default
spec:
  ai:
    enabled: true
    model: ggml-gpt4all-j
    backend: localai
    baseUrl: http://local-ai.svc.cluster.local:8080/v1
  noCache: false
  # k8sgpt 镜像tag
  version: v0.3.8
  filters:
  - Pod

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

$ kubectl get result -A
NAMESPACE   NAME                KIND   BACKEND
default     apisixapisixetcd2   Pod    localai
$ kubectl get result apisixapisixetcd2 -o yaml
apiVersion: core.k8sgpt.ai/v1alpha1
kind: Result
metadata:
  creationTimestamp: "2023-08-22T06:01:03Z"
  generation: 1
  name: apisixapisixetcd2
  namespace: default
  resourceVersion: "57982314"
  uid: b9c4e004-0655-4a9c-80af-fcfb00c976a7
spec:
  backend: localai
  details: "Example:  Error: Could not start container etcd pod=api-sx-etcdd(6e3a8b48-4f11-49c6-9a15-2f4e3c6c9d1b)
    with exit code: 1\nThe error message suggests that the container for etcd pod
    is failing with exit code 1. The error message also indicates that the pod is
    being restarted due to a backoff failure. \nThe solution suggests that the error
    could be caused by a few possible reasons, such as incorrect configuration of
    etcd or networking issues. \nThe steps to resolve the error are not specified
    in this message."
  error:
  - text: back-off 5m0s restarting failed container=etcd pod=apisix-etcd-2_apisix(d84a9fe9-f02a-41d2-a5a8-fd8dd859603b)
  kind: Pod
  name: apisix/apisix-etcd-2
  parentObject: ""
status: {}