{
"instances": [
{
"name": "testllama7b222",
"spec": {
"cluster": {
"subscription": "b4f61f4c-8a6a-4fdb-b4c1-f3b9a4aa4b24",
"resource_group": "default",
"zone": "cn-sh-01z",
"workspace": "c80f54fa-87a7-4405-a5f3-6222da897a58",
"aec2_name": "a69f1e6f-a620-46b1-8930-cef3c71f4d06"
},
"model": {
"name": "Llama-2-7b-hf",
"ref": {
"tag": "",
"branch": "master",
"hash": "1ad701afcbff1f92776334eb247a75e98bd3e0d8"
},
"type": "TYPE_AMP_PUBLIC_MODEL"
},
"framework": {
"name": "ams.inference_framework.cci",
"display_name": "ams.inference_framework.cci",
"version": "v0.0.3",
"image_repository": "registry.sensecore.tech/studio-ams/lightllm",
"image_tag": "lightllm-1.4.1-5a6ddfc",
"commands": [
"--max-input-length",
"2048",
"--max-total-tokens",
"3072",
"--max-batch-size",
"38",
"--max-concurrent-requests",
"128"
],
"envs": [],
"image_type": "IMAGE_TYPE_OFFICIAL",
"sidecar": null
},
"replicas": {
"value": 1
},
"resources": {
"cpu": "12",
"memory": "120",
"acceleration_hardwares": [
{
"type": "TYPE_GPU_NVIDIA",
"model": "MODEL_GPU_NVIDIA_A100",
"count": "1"
}
],
"cci_sku_name": "N2lS.Ie.I60.1"
},
"volume_mounts": [],
"zone": "cn-sh-01a"
},
"status": {
"render": {
"success": true,
"reasons": []
},
"health": {
"message": "This instance is healthy",
"status": "HEALTH_STATE_CODE_HEALTHY"
},
"resource_group": {
"subscription": "b4f61f4c-8a6a-4fdb-b4c1-f3b9a4aa4b24",
"resource_group": "default",
"zone": "cn-sh-01z",
"workspace": "c80f54fa-87a7-4405-a5f3-6222da897a58",
"name": "testllama7b222-8df62be1",
"resources": [
{
"name": "testllama7b222-8df62be1",
"type": "aec2/secrets",
"content": "{\"metadata\":{\"uid\":\"080cf6ab-b1d9-4a9c-bc5f-b90bd59c0be2\",\"deleted\":false,\"create_time\":\"2024-01-16T03:11:48.012068Z\",\"update_time\":\"2024-01-16T03:12:02.473820Z\"},\"ownership\":{\"creator_id\":\"0629ad1a-3784-4754-af90-948032f41322\",\"owner_id\":\"0629ad1a-3784-4754-af90-948032f41322\",\"tenant_id\":\"b4f61f4c-8a6a-4fdb-b4c1-f3b9a4aa4b24\",\"creator_name\":\"maasroot\"},\"info\":{\"subscription_name\":\"b4f61f4c-8a6a-4fdb-b4c1-f3b9a4aa4b24\",\"resource_group_name\":\"default\",\"zone\":\"cn-sh-01z\",\"workspace_name\":\"c80f54fa-87a7-4405-a5f3-6222da897a58\",\"workspace_display_name\":\"AMS_WS_3\",\"name\":\"testllama7b222-8df62be1\"},\"state\":\"CREATED\",\"secret_data\":[{\"key\":\"model_download_config.yaml\",\"value\":\"bG9nOgogIGxvZ19wYXRoOiAiLi9sb2dzL2RlZmF1bHQubG9nIgogIGxvZ19sZXZlbDogIkRFQlVHIgppZGVudGl0eToKICB1c2VySWQ6ICIwNjI5YWQxYS0zNzg0LTQ3NTQtYWY5MC05NDgwMzJmNDEzMjIiCiAgdGVuYW50SWQ6ICJiNGY2MWY0Yy04YTZhLTRmZGItYjRjMS1mM2I5YTRhYTRiMjQiCiAgdGVuYW50Q29kZTogIm1hYXNyb290Igptb2RlbDoKICBuYW1lOiAiTGxhbWEtMi03Yi1oZiIKICBlbnY6ICJ0ZWNoIgogIGxvY2FsX2RpcjogIi90bXAvc2hhcmVkLXN0b3JhZ2UvdGVzdGxsYW1hN2IyMjItOGRmNjJiZTEvMWFkNzAxYWZjYmZmMWY5Mjc3NjMzNGViMjQ3YTc1ZTk4YmQzZTBkOCIKICByZXZpc2lvbjogIjFhZDcwMWFmY2JmZjFmOTI3NzYzMzRlYjI0N2E3NWU5OGJkM2UwZDgiCiAgaXNfdXNlcjogZmFsc2UKICBpc19wdWJsaWM6IHRydWUKICBpc19maW5ldHVuZTogZmFsc2UKc2VjcmV0czogIkwzcEJiQ1VvM091RXloWHNBa0hyR3haclNNMHhRYmlJY2owRURmdE9zelM0UFRkcnhleHhtOU1iQXRJdmpCYWF2NEdLTmdxV0pWQlo0Sk5sczJPVUR6YWtSUlFEcE5QaFAxNmc3dURyK2lvMWRVNkdvQytoT2p0WjBkZDU5bDZHUDlYazM5NUdXVDNnenpWYnMxa1pxMEU3Y0ZEYS84RFRmM2E3NHduUGY3UjVmU093dnprbXoyWm5aWXRDYWNNYWRIMjVxU0IwQkg0N0dscWovOUhhRFVsUTR5RlFPbnl4ZWhjQ1Fqdy8wT2pPRWQwT1FFRW5XbHJtc1pXbXJoM0g2UUdlaG5DeVRlVzA5RWwvRmd1Mlg4UGJBNHg2bVNNZFdydnducHFpWGFjSGgzZUtRMTd5Ulg1M3U2VHJzcUE2elJxYkNJaDZWc2orQk10dENBZ211bWRWQlpaT3ZRRmpRWjdUNjF1eS82dEE1Ri8xdFFxNGc3b0Qwa0RLUzNBaHZKSDZ3VWNuNzFNSmY2SEVPSCtMKzhveGpIRnMxdkZTaE9peVRqYUhiVzFIL2lYUlhlUUFrTVBTSXk2NzFlTytOZmxVWmhSM0xZUy9GaHhjT0FEV2o1RytHQzRveUtlcFNVdkFwZDU3b2JneldOOU9MYjRYN0E4T0oyMUVuaTZZIgpyZXNvdXJjZV91aWQ6ICJjOWY4MzRiMi1iMTI2LTExZWUtOTgxMi1mMjZiMTRlMzM0MTUiCnJlc291cmNlOgogIGNwdVJlcXVlc3ROdW06ICIxMCIK\"}],\"secret_status\":{\"relatedResource\":[{\"resourceType\":\"CDI\",\"resourceName\":\"app-8df62be1\"}]}}"
},
{
"name": "testllama7b222-8df62be1",
"type": "aec2/configmaps",
"content": "{\"metadata\":{\"uid\":\"ea74a3f0-4dec-4fbf-8420-51317303d7f8\",\"deleted\":false,\"create_time\":\"2024-01-16T03:11:48.107224Z\",\"update_time\":\"2024-01-16T03:12:02.473820Z\"},\"ownership\":{\"creator_id\":\"0629ad1a-3784-4754-af90-948032f41322\",\"owner_id\":\"0629ad1a-3784-4754-af90-948032f41322\",\"tenant_id\":\"b4f61f4c-8a6a-4fdb-b4c1-f3b9a4aa4b24\",\"creator_name\":\"maasroot\"},\"info\":{\"subscription_name\":\"b4f61f4c-8a6a-4fdb-b4c1-f3b9a4aa4b24\",\"resource_group_name\":\"default\",\"zone\":\"cn-sh-01z\",\"workspace_name\":\"c80f54fa-87a7-4405-a5f3-6222da897a58\",\"workspace_display_name\":\"AMS_WS_3\",\"name\":\"testllama7b222-8df62be1\"},\"state\":\"CREATED\",\"configmap_data\":[{\"key\":\"metric-forward.yaml\",\"value\":\"gateway: \\\"https://tk-pushgateway-ams.cn-sh-01.sensecore.tech\\\"\\nstatic_configs:\\n - job_name: \\\"ams-inference-backend\\\"\\n scrape_interval: 15\\n scrape_timeout: 10\\n metrics_path: \\\"/metrics\\\"\\n scheme: \\\"http\\\"\\n target: \\\"localhost:18001\\\"\\n - job_name: \\\"ams-inference-sidecar\\\"\\n scrape_interval: 15\\n scrape_timeout: 10\\n metrics_path: \\\"/metrics\\\"\\n scheme: \\\"http\\\"\\n target: \\\"localhost:8002\\\"\\n\"},{\"key\":\"sidecar_config.yaml\",\"value\":\"server:\\n httpProxy:\\n addr: \\\"0.0.0.0:8080\\\"\\n internal:\\n addr: \\\"0.0.0.0:28080\\\"\\nlog:\\n mode: dev\\nmetric:\\n enable: true\\n addr: \\\"0.0.0.0:8002\\\"\\nservice:\\n discovery:\\n headlessSvcResolver:\\n enable: false\\n control:\\n pool:\\n size: 1000\\n maxWait: 100\\n interval: 100ms\\n tokenLoad:\\n threshold: 0.999\\n endpoint: http://127.0.0.1:18001\\n path: \\\"\\\"\\n timeout: 1s\\n interval: 100ms\\n switchInterval: 100ms\\n mq:\\n enable: true\\n control:\\n topic: \\\"maasroot_testllama7b\\\"\\n groupId: \\\"\\\"\\n workerNum: 1000\\n data:\\n transport:\\n mqProxyAddr:\\n addr: \\\"ws://ams-mq-proxy-data.sensecoreapi.tech:33200\\\"\\n\"}],\"configmap_status\":{\"relatedResource\":[{\"resourceType\":\"CDI\",\"resourceName\":\"app-8df62be1\"}]}}"
},
{
"name": "app-8df62be1",
"type": "aec2/apps",
"content": "{\"metadata\":{\"uid\":\"368axjuv6uolfyxclliq4t6efxxs8vhk8cpd\",\"deleted\":false,\"create_time\":\"2024-01-16T03:11:51.268973Z\",\"update_time\":\"2024-01-16T03:11:51.268973Z\"},\"ownership\":{\"creator_id\":\"0629ad1a-3784-4754-af90-948032f41322\",\"owner_id\":\"0629ad1a-3784-4754-af90-948032f41322\",\"tenant_id\":\"b4f61f4c-8a6a-4fdb-b4c1-f3b9a4aa4b24\",\"creator_name\":\"maasroot\"},\"info\":{\"subscription_name\":\"b4f61f4c-8a6a-4fdb-b4c1-f3b9a4aa4b24\",\"resource_group_name\":\"default\",\"zone\":\"cn-sh-01z\",\"workspace_name\":\"c80f54fa-87a7-4405-a5f3-6222da897a58\",\"workspace_display_name\":\"AMS_WS_3\",\"aec2_name\":\"a69f1e6f-a620-46b1-8930-cef3c71f4d06\",\"aec2_id\":\"24da17b6-a611-11ee-ac7e-ca487d28e0c1\",\"vpc_id\":\"77090356-c95a-11ed-a770-0e34dfd438bb\",\"name\":\"app-8df62be1\",\"display_name\":\"testllama7b222-8df62be1\",\"description\":\"Generated. Don't Delete Or Change.\\n由模型推理服务生成，请勿改动或删除。\"},\"template\":{\"init_containers\":[{\"name\":\"model-download\",\"image_type\":\"PRIVATE\",\"image\":\"registry.sensecore.tech/studio-ams/ams-model-downloader:7c245fa\",\"image_display_name\":\"\",\"command\":[],\"env\":[],\"need_cache\":false,\"container_type\":\"INIT\",\"resource_request\":{\"cpu\":\"10\",\"memory\":\"118GiB\"},\"volume_mounts\":[{\"type\":\"Secret\",\"name\":\"testllama7b222-8df62be1\",\"mount_path\":\"/root/run/config\",\"afs_id\":\"\",\"volume_capacity\":\"\",\"mount_afs_subdir\":\"\"}],\"livenessProbe\":null}],\"containers\":[{\"name\":\"backend\",\"image_type\":\"PRIVATE\",\"image\":\"registry.sensecore.tech/studio-ams/lightllm:lightllm-1.4.1-5a6ddfc\",\"image_display_name\":\"\",\"command\":[\"text-generation-launcher\",\"--model-id\",\"/tmp/shared-storage/testllama7b222-8df62be1/1ad701afcbff1f92776334eb247a75e98bd3e0d8/model\",\"--num-shard\",\"1\",\"--trust-remote-code\",\"--max-input-length\",\"2048\",\"--max-total-tokens\",\"3072\",\"--max-batch-size\",\"38\",\"--max-concurrent-requests\",\"128\",\"--port\",\"18001\"],\"env\":[{\"name\":\"LIGHTLLM\",\"value\":\"True\",\"need_mosaic\":false}],\"need_cache\":false,\"container_type\":\"MAIN\",\"resource_request\":{\"cpu\":\"10\",\"memory\":\"118GiB\",\"nvidia.com/gpu\":\"1\"},\"volume_mounts\":[],\"livenessProbe\":null},{\"name\":\"metrics\",\"image_type\":\"PRIVATE\",\"image\":\"registry.sensecore.tech/studio-ams/ams-metrics-sidecar:v0.1.6\",\"image_display_name\":\"\",\"command\":[],\"env\":[{\"name\":\"CONFIG_PATH\",\"value\":\"/config/metric-forward.yaml\",\"need_mosaic\":false},{\"name\":\"RESOURCE__RESOURCEMANAGER_ID\",\"value\":\"c9f834b2-b126-11ee-9812-f26b14e33415\",\"need_mosaic\":false},{\"name\":\"TOPIC\",\"value\":\"maasroot_testllama7b\",\"need_mosaic\":false}],\"need_cache\":false,\"container_type\":\"MAIN\",\"resource_request\":{\"cpu\":\"1\",\"memory\":\"1GiB\"},\"volume_mounts\":[{\"type\":\"Configmap\",\"name\":\"testllama7b222-8df62be1\",\"mount_path\":\"/config\",\"afs_id\":\"\",\"volume_capacity\":\"\",\"mount_afs_subdir\":\"\"}],\"livenessProbe\":null},{\"name\":\"sidecar\",\"image_type\":\"PRIVATE\",\"image\":\"registry.sensecore.tech/studio-ams/ams-inference-sidecar:0.0.1-cbbc8c6\",\"image_display_name\":\"\",\"command\":[\"sh\",\"-c\",\"./bin/ams-inference-sidecar -conf /config/sidecar_config.yaml -autoReloadSecond 10\"],\"env\":[],\"need_cache\":false,\"container_type\":\"MAIN\",\"resource_request\":{\"cpu\":\"1\",\"memory\":\"1GiB\"},\"volume_mounts\":[{\"type\":\"Configmap\",\"name\":\"testllama7b222-8df62be1\",\"mount_path\":\"/config\",\"afs_id\":\"\",\"volume_capacity\":\"\",\"mount_afs_subdir\":\"\"}],\"livenessProbe\":null}],\"compute_resource_spec\":{\"name\":\"N2lS.Ie.I60.1\"}},\"scaling\":{\"scaling_strategy\":\"\",\"replicas\":1,\"cpu_load\":0,\"min_replicas\":0,\"max_replicas\":0},\"networking\":{\"access_port\":0,\"intranet_ip\":\"10.96.170.171\",\"intranet_domain_name\":\"app-8df62be1.ns-operations-5edb2e3a.svc.cluster.local\"},\"scheduling\":{\"priority\":\"HIGHEST\",\"cluster_id\":\"cn-sh-01a\",\"resource_pool_id\":\"24da17b6-a611-11ee-ac7e-ca487d28e0c1\"},\"ready_replicas\":1,\"state\":\"RUNNING\",\"transition_error\":\"\",\"need_cache\":false,\"app_port\":[18001],\"execution_type\":\"START\",\"aec2\":{\"id\":\"/subscriptions/b4f61f4c-8a6a-4fdb-b4c1-f3b9a4aa4b24/resourceGroups/default/zones/cn-sh-01a/aec2s/a69f1e6f-a620-46b1-8930-cef3c71f4d06\",\"uid\":\"24da17b6-a611-11ee-ac7e-ca487d28e0c1\",\"name\":\"a69f1e6f-a620-46b1-8930-cef3c71f4d06\",\"display_name\":\"AMS_3820\",\"state\":\"ACTIVE\",\"type\":\"\"},\"last_operation_type\":\"UPDATE\",\"resource_spec\":{\"id\":\"N2lS.Ie.I60.1\",\"name\":\"N2lS.Ie.I60.1\",\"description\":\"1XA100-80GB\",\"category\":\"JOB\",\"sku_id\":\"N2lS.Ie.I60\",\"cpu\":{\"manufacturer\":\"Intel\",\"type\":\"6348\",\"frequency\":2.6,\"vcpu_allocatable\":12,\"vcpu_capacity\":0},\"memory\":{\"allocatable\":120,\"capacity\":0},\"device\":{\"manufacturer\":\"NVIDIA\",\"type\":\"A100\",\"memory\":80,\"number\":1,\"physical_interface\":\"SXM4\"},\"training_network\":{\"network\":{\"type\":\"IB\",\"nics_number\":2,\"bandwidth\":\"200G\"},\"name\":\"\"},\"service_network\":{\"network\":{\"type\":\"\",\"nics_number\":0,\"bandwidth\":\"\"}},\"data_network\":{\"network\":{\"type\":\"\",\"nics_number\":0,\"bandwidth\":\"\"}},\"storage\":null,\"custom\":\"\"},\"strategy\":{\"maxSurge\":\"\",\"maxUnavailable\":\"\"}}"
}
]
},
"replicas": [
{
"name": "app-8df62be1-64c4c6857c-tpwt9",
"health": {
"message": "",
"status": "HEALTH_STATE_CODE_HEALTHY"
},
"owner": {
"name": "app-8df62be1",
"type": "aec2/apps",
"content": ""
}
}
]
}
}
]
}