先部署milvus, 使用单机模式模式
milvus-install.yaml
---
apiVersion: v1
kind: PersistentVolume
metadata:
annotations:
pv.kubernetes.io/bound-by-controller: "yes"
finalizers:
- kubernetes.io/pv-protection
managedFields:
- apiVersion: v1
fieldsType: FieldsV1
fieldsV1:
f:spec:
f:accessModes: {}
f:capacity:
.: {}
f:storage: {}
f:hostPath:
.: {}
f:path: {}
f:type: {}
f:persistentVolumeReclaimPolicy: {}
f:volumeMode: {}
manager: agent
operation: Update
- apiVersion: v1
fieldsType: FieldsV1
fieldsV1:
f:status:
f:phase: {}
manager: kube-controller-manager
operation: Update
subresource: status
- apiVersion: v1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.: {}
f:pv.kubernetes.io/bound-by-controller: {}
f:spec:
f:claimRef: {}
manager: kube-controller-manager
operation: Update
name: my-release-milvus-pv
spec:
accessModes:
- ReadWriteOnce
capacity:
storage: 50Gi
claimRef:
apiVersion: v1
kind: PersistentVolumeClaim
name: my-release-milvus
namespace: milvus
resourceVersion: "1844339"
uid: 7efe1c3c-bd92-4c21-9b4d-d5d99d06a835
hostPath:
path: /opt/pv/my-release-milvus-pv
type: DirectoryOrCreate
persistentVolumeReclaimPolicy: Retain
volumeMode: Filesystem
---
apiVersion: v1
kind: PersistentVolume
metadata:
annotations:
pv.kubernetes.io/bound-by-controller: "yes"
finalizers:
- kubernetes.io/pv-protection
managedFields:
- apiVersion: v1
fieldsType: FieldsV1
fieldsV1:
f:spec:
f:accessModes: {}
f:capacity:
.: {}
f:storage: {}
f:hostPath:
.: {}
f:path: {}
f:type: {}
f:persistentVolumeReclaimPolicy: {}
f:volumeMode: {}
manager: agent
operation: Update
time: "2024-08-16T02:08:43Z"
- apiVersion: v1
fieldsType: FieldsV1
fieldsV1:
f:status:
f:phase: {}
manager: kube-controller-manager
operation: Update
subresource: status
time: "2024-08-16T02:08:43Z"
- apiVersion: v1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.: {}
f:pv.kubernetes.io/bound-by-controller: {}
f:spec:
f:claimRef: {}
manager: kube-controller-manager
operation: Update
name: my-release-minio-pv
spec:
accessModes:
- ReadWriteOnce
capacity:
storage: 500Gi
claimRef:
apiVersion: v1
kind: PersistentVolumeClaim
name: my-release-minio
namespace: milvus
resourceVersion: "1844338"
uid: 5e332cae-55df-4a22-a673-94cd18e06bad
hostPath:
path: /opt/pv/my-release-minio-pv
type: DirectoryOrCreate
persistentVolumeReclaimPolicy: Retain
volumeMode: Filesystem
---
# Source: milvus/charts/minio/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: "my-release-minio"
namespace: "milvus"
labels:
app: minio
chart: minio-8.0.17
release: "my-release"
---
# Source: milvus/charts/minio/templates/secrets.yaml
apiVersion: v1
kind: Secret
metadata:
name: my-release-minio
namespace: "milvus"
labels:
app: minio
chart: minio-8.0.17
release: my-release
heritage: Helm
type: Opaque
data:
accesskey: "bWluaW9hZG1pbg=="
secretkey: "bWluaW9hZG1pbg=="
---
# Source: milvus/charts/minio/templates/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: my-release-minio
namespace: "milvus"
labels:
app: minio
chart: minio-8.0.17
release: my-release
heritage: Helm
data:
initialize: |-
#!/bin/sh
set -e ; # Have script exit in the event of a failed command.
MC_CONFIG_DIR="/etc/minio/mc/"
MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}"
# connectToMinio
# Use a check-sleep-check loop to wait for Minio service to be available
connectToMinio() {
SCHEME=$1
ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts
set -e ; # fail if we can't read the keys.
ACCESS=$(cat /config/accesskey) ; SECRET=$(cat /config/secretkey) ;
set +e ; # The connections to minio are allowed to fail.
echo "Connecting to Minio server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ;
MC_COMMAND="${MC} config host add myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ;
$MC_COMMAND ;
STATUS=$? ;
until [ $STATUS = 0 ]
do
ATTEMPTS=`expr $ATTEMPTS + 1` ;
echo \"Failed attempts: $ATTEMPTS\" ;
if [ $ATTEMPTS -gt $LIMIT ]; then
exit 1 ;
fi ;
sleep 2 ; # 1 second intervals between attempts
$MC_COMMAND ;
STATUS=$? ;
done ;
set -e ; # reset `e` as active
return 0
}
# checkBucketExists ($bucket)
# Check if the bucket exists, by using the exit code of `mc ls`
checkBucketExists() {
BUCKET=$1
CMD=$(${MC} ls myminio/$BUCKET > /dev/null 2>&1)
return $?
}
# createBucket ($bucket, $policy, $purge)
# Ensure bucket exists, purging if asked to
createBucket() {
BUCKET=$1
POLICY=$2
PURGE=$3
VERSIONING=$4
# Purge the bucket, if set & exists
# Since PURGE is user input, check explicitly for `true`
if [ $PURGE = true ]; then
if checkBucketExists $BUCKET ; then
echo "Purging bucket '$BUCKET'."
set +e ; # don't exit if this fails
${MC} rm -r --force myminio/$BUCKET
set -e ; # reset `e` as active
else
echo "Bucket '$BUCKET' does not exist, skipping purge."
fi
fi
# Create the bucket if it does not exist
if ! checkBucketExists $BUCKET ; then
echo "Creating bucket '$BUCKET'"
${MC} mb myminio/$BUCKET
else
echo "Bucket '$BUCKET' already exists."
fi
# set versioning for bucket
if [ ! -z $VERSIONING ] ; then
if [ $VERSIONING = true ] ; then
echo "Enabling versioning for '$BUCKET'"
${MC} version enable myminio/$BUCKET
elif [ $VERSIONING = false ] ; then
echo "Suspending versioning for '$BUCKET'"
${MC} version suspend myminio/$BUCKET
fi
else
echo "Bucket '$BUCKET' versioning unchanged."
fi
# At this point, the bucket should exist, skip checking for existence
# Set policy on the bucket
echo "Setting policy of bucket '$BUCKET' to '$POLICY'."
${MC} policy set $POLICY myminio/$BUCKET
}
# Try connecting to Minio instance
scheme=http
connectToMinio $scheme
---
# Source: milvus/templates/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: my-release-milvus
namespace: "milvus"
data:
default.yaml: |+
# Copyright (C) 2019-2021 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
etcd:
endpoints:
- my-release-etcd.milvus.svc.cluster.local:2379
metastore:
type: etcd
minio:
address: my-release-minio
port: 9000
accessKeyID: minioadmin
secretAccessKey: minioadmin
useSSL: false
bucketName: milvus-bucket
rootPath: file
useIAM: false
iamEndpoint:
region:
useVirtualHost: false
mq:
type: rocksmq
messageQueue: rocksmq
rootCoord:
address: localhost
port: 53100
enableActiveStandby: false # Enable rootcoord active-standby
proxy:
port: 19530
internalPort: 19529
queryCoord:
address: localhost
port: 19531
enableActiveStandby: false # Enable querycoord active-standby
queryNode:
port: 21123
enableDisk: true # Enable querynode load disk index, and search on disk index
indexCoord:
address: localhost
port: 31000
enableActiveStandby: false # Enable indexcoord active-standby
indexNode:
port: 21121
enableDisk: true # Enable index node build disk vector index
dataCoord:
address: localhost
port: 13333
enableActiveStandby: false # Enable datacoord active-standby
dataNode:
port: 21124
log:
level: info
file:
rootPath: ""
maxSize: 300
maxAge: 10
maxBackups: 20
format: text
user.yaml: |-
# For example enable rest http for milvus proxy
# proxy:
# http:
# enabled: true
---
# Source: milvus/charts/minio/templates/pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: my-release-minio
namespace: "milvus"
annotations:
helm.sh/resource-policy: keep
labels:
app: minio
chart: minio-8.0.17
release: my-release
heritage: Helm
spec:
accessModes:
- "ReadWriteOnce"
resources:
requests:
storage: "500Gi"
---
# Source: milvus/templates/pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: my-release-milvus
namespace: "milvus"
annotations:
helm.sh/resource-policy: keep
labels:
helm.sh/chart: milvus-4.1.8
app.kubernetes.io/name: milvus
app.kubernetes.io/instance: my-release
app.kubernetes.io/version: "2.3.2"
app.kubernetes.io/managed-by: Helm
spec:
accessModes:
- "ReadWriteOnce"
resources:
requests:
storage: 50Gi
---
# Source: milvus/charts/etcd/templates/svc-headless.yaml
apiVersion: v1
kind: Service
metadata:
name: my-release-etcd-headless
namespace: milvus
labels:
app.kubernetes.io/name: etcd
helm.sh/chart: etcd-6.3.3
app.kubernetes.io/instance: my-release
app.kubernetes.io/managed-by: Helm
annotations:
service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
spec:
type: ClusterIP
clusterIP: None
publishNotReadyAddresses: true
ports:
- name: "client"
port: 2379
targetPort: client
- name: "peer"
port: 2380
targetPort: peer
selector:
app.kubernetes.io/name: etcd
app.kubernetes.io/instance: my-release
---
# Source: milvus/charts/etcd/templates/svc.yaml
apiVersion: v1
kind: Service
metadata:
name: my-release-etcd
namespace: milvus
labels:
app.kubernetes.io/name: etcd
helm.sh/chart: etcd-6.3.3
app.kubernetes.io/instance: my-release
app.kubernetes.io/managed-by: Helm
annotations:
spec:
type: ClusterIP
ports:
- name: "client"
port: 2379
targetPort: 2379
nodePort: null
- name: "peer"
port: 2380
targetPort: 2380
nodePort: null
selector:
app.kubernetes.io/name: etcd
app.kubernetes.io/instance: my-release
---
# Source: milvus/charts/minio/templates/service.yaml
apiVersion: v1
kind: Service
metadata:
name: my-release-minio
namespace: milvus
labels:
app: minio
chart: minio-8.0.17
release: my-release
heritage: Helm
spec:
type: ClusterIP
ports:
- name: http
port: 9000
protocol: TCP
targetPort: 9000
selector:
app: minio
release: my-release
---
# Source: milvus/templates/service.yaml
apiVersion: v1
kind: Service
metadata:
name: my-release-milvus
namespace: milvus
labels:
helm.sh/chart: milvus-4.1.8
app.kubernetes.io/name: milvus
app.kubernetes.io/instance: my-release
app.kubernetes.io/version: "2.3.2"
app.kubernetes.io/managed-by: Helm
component: "standalone"
spec:
type: ClusterIP
ports:
- name: milvus
port: 19530
protocol: TCP
targetPort: milvus
- name: metrics
protocol: TCP
port: 9091
targetPort: metrics
selector:
app.kubernetes.io/name: milvus
app.kubernetes.io/instance: my-release
component: "standalone"
---
# Source: milvus/charts/minio/templates/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: my-release-minio
namespace: milvus
labels:
app: minio
chart: minio-8.0.17
release: my-release
heritage: Helm
spec:
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 100%
maxUnavailable: 0
selector:
matchLabels:
app: minio
release: my-release
template:
metadata:
name: my-release-minio
labels:
app: minio
release: my-release
annotations:
checksum/secrets: aa3f4f64eb45653d3fee45079a6e55a9869ce76297baa50df3bc33192434d05e
checksum/config: ed4d4467dd70f3e0ed89e5d2bc3c4414b02a52fcf7db7f1b66b675b9016664a7
spec:
serviceAccountName: "my-release-minio"
securityContext:
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
containers:
- name: minio
image: "minio/minio:RELEASE.2023-03-20T20-16-18Z"
imagePullPolicy: IfNotPresent
command:
- "/bin/sh"
- "-ce"
- "/usr/bin/docker-entrypoint.sh minio -S /etc/minio/certs/ server /export"
volumeMounts:
- name: export
mountPath: /export
ports:
- name: http
containerPort: 9000
livenessProbe:
httpGet:
path: /minio/health/live
port: http
scheme: HTTP
initialDelaySeconds: 5
periodSeconds: 5
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
readinessProbe:
tcpSocket:
port: http
initialDelaySeconds: 5
periodSeconds: 5
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 5
startupProbe:
tcpSocket:
port: http
initialDelaySeconds: 0
periodSeconds: 10
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 60
env:
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
name: my-release-minio
key: accesskey
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
name: my-release-minio
key: secretkey
resources:
requests:
memory: 2Gi
volumes:
- name: export
persistentVolumeClaim:
claimName: my-release-minio
- name: minio-user
secret:
secretName: my-release-minio
---
# Source: milvus/templates/standalone-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: my-release-milvus-standalone
namespace: milvus
labels:
helm.sh/chart: milvus-4.1.8
app.kubernetes.io/name: milvus
app.kubernetes.io/instance: my-release
app.kubernetes.io/version: "2.3.2"
app.kubernetes.io/managed-by: Helm
component: "standalone"
annotations:
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app.kubernetes.io/name: milvus
app.kubernetes.io/instance: my-release
component: "standalone"
template:
metadata:
labels:
app.kubernetes.io/name: milvus
app.kubernetes.io/instance: my-release
component: "standalone"
annotations:
checksum/config: bc6a7e82027efa3ad0df3b00d07c3ad1dcc40d7665f314e6d13a000fdd26aec2
spec:
serviceAccountName: default
initContainers:
- name: config
command:
- /cp
- /run-helm.sh,/merge
- /milvus/tools/run-helm.sh,/milvus/tools/merge
image: "milvusdb/milvus-config-tool:v0.1.1"
imagePullPolicy: IfNotPresent
volumeMounts:
- mountPath: /milvus/tools
name: tools
containers:
- name: standalone
image: "milvusdb/milvus:v2.3.21"
imagePullPolicy: IfNotPresent
args: [ "/milvus/tools/run-helm.sh", "milvus", "run", "standalone" ]
ports:
- name: milvus
containerPort: 19530
protocol: TCP
- name: metrics
containerPort: 9091
protocol: TCP
livenessProbe:
httpGet:
path: /healthz
port: metrics
initialDelaySeconds: 90
periodSeconds: 30
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
readinessProbe:
httpGet:
path: /healthz
port: metrics
initialDelaySeconds: 90
periodSeconds: 10
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
resources:
{}
env:
volumeMounts:
- mountPath: /milvus/tools
name: tools
- name: milvus-config
mountPath: /milvus/configs/default.yaml
subPath: default.yaml
readOnly: true
- name: milvus-config
mountPath: /milvus/configs/user.yaml
subPath: user.yaml
readOnly: true
- name: milvus-data-disk
mountPath: "/var/lib/milvus"
subPath:
- mountPath: /var/lib/milvus/data
name: disk
volumes:
- emptyDir: {}
name: tools
- name: milvus-config
configMap:
name: my-release-milvus
- name: milvus-data-disk
persistentVolumeClaim:
claimName: my-release-milvus
- name: disk
emptyDir: {}
---
# Source: milvus/charts/etcd/templates/statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: my-release-etcd
namespace: milvus
labels:
app.kubernetes.io/name: etcd
helm.sh/chart: etcd-6.3.3
app.kubernetes.io/instance: my-release
app.kubernetes.io/managed-by: Helm
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: etcd
app.kubernetes.io/instance: my-release
serviceName: my-release-etcd-headless
podManagementPolicy: Parallel
updateStrategy:
type: RollingUpdate
template:
metadata:
labels:
app.kubernetes.io/name: etcd
helm.sh/chart: etcd-6.3.3
app.kubernetes.io/instance: my-release
app.kubernetes.io/managed-by: Helm
annotations:
spec:
affinity:
podAffinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
app.kubernetes.io/name: etcd
app.kubernetes.io/instance: my-release
namespaces:
- "default"
topologyKey: kubernetes.io/hostname
weight: 1
nodeAffinity:
securityContext:
fsGroup: 1001
serviceAccountName: "default"
containers:
- name: etcd
image: docker.io/milvusdb/etcd:3.5.5-r4
imagePullPolicy: "IfNotPresent"
securityContext:
runAsNonRoot: true
runAsUser: 1001
env:
- name: BITNAMI_DEBUG
value: "false"
- name: MY_POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: MY_POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: ETCDCTL_API
value: "3"
- name: ETCD_ON_K8S
value: "yes"
- name: ETCD_START_FROM_SNAPSHOT
value: "no"
- name: ETCD_DISASTER_RECOVERY
value: "no"
- name: ETCD_NAME
value: "$(MY_POD_NAME)"
- name: ETCD_DATA_DIR
value: "/bitnami/etcd/data"
- name: ETCD_LOG_LEVEL
value: "info"
- name: ALLOW_NONE_AUTHENTICATION
value: "yes"
- name: ETCD_ADVERTISE_CLIENT_URLS
value: "http://$(MY_POD_NAME).my-release-etcd-headless.milvus.svc.cluster.local:2379"
- name: ETCD_LISTEN_CLIENT_URLS
value: "http://0.0.0.0:2379"
- name: ETCD_INITIAL_ADVERTISE_PEER_URLS
value: "http://$(MY_POD_NAME).my-release-etcd-headless.milvus.svc.cluster.local:2380"
- name: ETCD_LISTEN_PEER_URLS
value: "http://0.0.0.0:2380"
- name: ETCD_AUTO_COMPACTION_MODE
value: "revision"
- name: ETCD_AUTO_COMPACTION_RETENTION
value: "1000"
- name: ETCD_QUOTA_BACKEND_BYTES
value: "4294967296"
- name: ETCD_HEARTBEAT_INTERVAL
value: "500"
- name: ETCD_ELECTION_TIMEOUT
value: "2500"
envFrom:
ports:
- name: client
containerPort: 2379
protocol: TCP
- name: peer
containerPort: 2380
protocol: TCP
livenessProbe:
exec:
command:
- /opt/bitnami/scripts/etcd/healthcheck.sh
initialDelaySeconds: 60
periodSeconds: 30
timeoutSeconds: 10
successThreshold: 1
failureThreshold: 5
readinessProbe:
exec:
command:
- /opt/bitnami/scripts/etcd/healthcheck.sh
initialDelaySeconds: 60
periodSeconds: 20
timeoutSeconds: 10
successThreshold: 1
failureThreshold: 5
resources:
limits: {}
requests: {}
volumeMounts:
- name: data
mountPath: /bitnami/etcd
volumes:
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes:
- "ReadWriteOnce"
resources:
requests:
storage: "10Gi"
创建图形管理界面工具attu
attu-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: milvus-attu
namespace: milvus
spec:
replicas: 1
selector:
matchLabels:
app: milvus-attu
template:
metadata:
labels:
app: milvus-attu
spec:
containers:
- name: milvus-attu
image: zilliz/attu:latest
ports:
- containerPort: 3000
env:
- name: MILVUS_HOST
value: 192.168.110.125
- name: MILVUS_PORT
value: "19530"
---
apiVersion: v1
kind: Service
metadata:
name: milvus-attu
namespace: milvus
spec:
type: NodePort
ports:
- port: 3000
targetPort: 3000
nodePort: 31333
selector:
app: milvus-attu
先录入点数据然后调用查询接口查看相似度
import time
from pymilvus import connections, FieldSchema, CollectionSchema, Collection, DataType, MilvusException
# 连接到 Milvus 服务
host = '192.168.110.125'
port = 19530
connections.connect(alias='default', host=host, port=port)
# 定义字段
fields = [
FieldSchema(name="id", dtype=DataType.INT64, is_primary=True),
FieldSchema(name="vec", dtype=DataType.FLOAT_VECTOR, dim=128)
]
# 创建集合
schema = CollectionSchema(fields=fields, description="test collection")
collection = Collection(name="test", schema=schema)
# 生成一些测试数据
import numpy as np
vectors = [[np.random.random() for _ in range(128)] for _ in range(10)]
ids = list(range(10))
# 插入数据
collection.insert(data=[ids, vectors])
print(collection.describe())
query_vetor = np.random.random(128).astype(np.float32)
search_params = {
"metric_type": "COSINE",
"params": {
"nprobe": 10
}
}
results = None
try:
# 执行搜索操作
results = collection.search(
data=[query_vetor],
anns_field="vec",
param=search_params,
limit=5,
output_fields=["id", "vec"]
)
except MilvusException as e:
print(f"Search error: {e}")
# 增加重试次数和间隔时间
retries = 3
delay = 1
for attempt in range(retries):
try:
results = collection.search(
data=[query_vetor],
anns_field="vec",
param=search_params,
limit=5,
output_fields=["id", "vec"]
)
break
except MilvusException as e:
print(f"Retry attempt {attempt+1}/{retries}: {e}")
time.sleep(delay)
# 5. 打印搜索结果
if results:
for res in results[0]:
print(f"id: {res.id}, distance: {res.distance}")
else:
print("no foud result")
# 6. 关闭连接
connections.disconnect(alias='default')