Anyscale

Portkey 帮助将 Anyscale API 推向生产环境，提供可观察性、回退、缓存等抽象。通过 Portkey 使用 Anyscale API 以实现：

1. 增强日志记录：通过详细洞察跟踪 API 使用情况。

2. 生产可靠性：自动回退、负载均衡和缓存。

3. 持续改进：收集并应用用户反馈。

4. 增强微调：结合日志和用户反馈进行有针对性的微调。

1.1 设置与日志记录

1. 设置 $ export OPENAI_API_KEY=ANYSCALE_API_KEY

2. 获取您的 Portkey API 密钥。

3. 切换到 Portkey 网关 URL: https://api.portkey.ai/v1/proxy

查看请求的完整日志（延迟、费用、令牌）——并通过他们的分析套件深入挖掘数据。

""" OPENAI PYTHON SDK """
import openai

PORTKEY_GATEWAY_URL = "https://api.portkey.ai/v1"

PORTKEY_HEADERS = {
	'Authorization': 'Bearer ANYSCALE_KEY',
	'Content-Type': 'application/json',
	# **************************************
	'x-portkey-api-key': 'PORTKEY_API_KEY', 	# Get from https://app.portkey.ai/,
	'x-portkey-provider': 'anyscale' 		# Tell Portkey that the request is for Anyscale
	# **************************************
}

client = openai.OpenAI(base_url=PORTKEY_GATEWAY_URL, default_headers=PORTKEY_HEADERS)

response = client.chat.completions.create(
    model="mistralai/Mistral-7B-Instruct-v0.1",
    messages=[{"role": "user", "content": "Say this is a test"}]
)

print(response.choices[0].message.content)

""" OPENAI NODE SDK """
import OpenAI from 'openai';

const PORTKEY_GATEWAY_URL = "https://api.portkey.ai/v1"

const PORTKEY_HEADERS = {
	'Authorization': 'Bearer ANYSCALE_KEY',
	'Content-Type': 'application/json',
	// **************************************
	'x-portkey-api-key': 'PORTKEY_API_KEY', 	// Get from https://app.portkey.ai/,
	'x-portkey-provider': 'anyscale' 		// Tell Portkey that the request is for Anyscale
	// **************************************
}

const openai = new OpenAI({baseURL:PORTKEY_GATEWAY_URL, defaultHeaders:PORTKEY_HEADERS});

async function main() {
  const chatCompletion = await openai.chat.completions.create({
    messages: [{ role: 'user', content: 'Say this is a test' }],
    model: 'mistralai/Mistral-7B-Instruct-v0.1',
  });
  console.log(chatCompletion.choices[0].message.content);
}

main();

""" REQUESTS LIBRARY """
import requests

PORTKEY_GATEWAY_URL = "https://api.portkey.ai/v1/chat/completions"

PORTKEY_HEADERS = {
	'Authorization': 'Bearer ANYSCALE_KEY',
	'Content-Type': 'application/json',
	# **************************************
	'x-portkey-api-key': 'PORTKEY_API_KEY', 	# Get from https://app.portkey.ai/,
	'x-portkey-provider': 'anyscale' 		# Tell Portkey that the request is for Anyscale
	# **************************************
}

DATA = {
    "messages": [{"role": "user", "content": "What happens when you mix red & yellow?"}],
    "model": "mistralai/Mistral-7B-Instruct-v0.1"
}

response = requests.post(PORTKEY_GATEWAY_URL, headers=PORTKEY_HEADERS, json=DATA)

print(response.text)

""" CURL """
curl "https://api.portkey.ai/v1/chat/completions" \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer ANYSCALE_KEY" \
  -H "x-portkey-api-key: PORTKEY_API_KEY" \
  -H "x-portkey-provider: anyscale" \
  -d '{
    "model": "meta-llama/Llama-2-70b-chat-hf",
    "messages": [{"role": "user", "content": "Say 'Test'."}]
  }'

1.2. 增强可观察性

• 跟踪 单个 ID 的请求。

• 附加自定义标签 以进行请求分段和深入分析。

只需将相关的头部添加到您的请求中：

""" OPENAI PYTHON SDK """
import json, openai

PORTKEY_GATEWAY_URL = "https://api.portkey.ai/v1"

TRACE_ID = 'anyscale_portkey_test'

METADATA = {
    "_environment": "production",
    "_user": "userid123",
    "_organisation": "orgid123",
    "_prompt": "summarisationPrompt"
}

PORTKEY_HEADERS = {
	'Authorization': 'Bearer ANYSCALE_KEY',
	'Content-Type': 'application/json',
	'x-portkey-api-key': 'PORTKEY_API_KEY',
	'x-portkey-provider': 'anyscale',
	# **************************************
	'x-portkey-trace-id': TRACE_ID, 		# 发送跟踪 ID
	'x-portkey-metadata': json.dumps(METADATA) 	# 发送元数据
	# **************************************
}

client = openai.OpenAI(base_url=PORTKEY_GATEWAY_URL, default_headers=PORTKEY_HEADERS)

response = client.chat.completions.create(
	model="mistralai/Mistral-7B-Instruct-v0.1",
	messages=[{"role": "user", "content": "Say this is a test"}]
)

print(response.choices[0].message.content)

""" OPENAI NODE SDK """
import OpenAI from 'openai';

const PORTKEY_GATEWAY_URL = "https://api.portkey.ai/v1"

const TRACE_ID = 'anyscale_portkey_test'

const METADATA = {
    "_environment": "production",
    "_user": "userid123",
    "_organisation": "orgid123",
    "_prompt": "summarisationPrompt"
}

const PORTKEY_HEADERS = {
	'Authorization': 'Bearer ANYSCALE_KEY',
	'Content-Type': 'application/json',
	'x-portkey-api-key': 'PORTKEY_API_KEY',
	'x-portkey-provider': 'anyscale',
	// **************************************
	'x-portkey-trace-id': TRACE_ID, 		// 发送跟踪 ID
	'x-portkey-metadata': JSON.stringify(METADATA) 	// 发送元数据
	// **************************************
}

const openai = new OpenAI({baseURL:PORTKEY_GATEWAY_URL, defaultHeaders:PORTKEY_HEADERS});

async function main() {
  const chatCompletion = await openai.chat.completions.create({
    messages: [{ role: 'user', content: 'Say this is a test' }],
    model: 'mistralai/Mistral-7B-Instruct-v0.1',
  });
  console.log(chatCompletion.choices[0].message.content);
}

main();

""" REQUESTS LIBRARY """
import requests, json

PORTKEY_GATEWAY_URL = "https://api.portkey.ai/v1/chat/completions"

TRACE_ID = 'anyscale_portkey_test'

METADATA = {
    "_environment": "production",
    "_user": "userid123",
    "_organisation": "orgid123",
    "_prompt": "summarisationPrompt"
}

PORTKEY_HEADERS = {
	'Authorization': 'Bearer ANYSCALE_KEY',
	'Content-Type': 'application/json',
	'x-portkey-api-key': 'PORTKEY_API_KEY',
	'x-portkey-provider': 'anyscale',
	# **************************************
	'x-portkey-trace-id': TRACE_ID, 		# 发送跟踪 ID
	'x-portkey-metadata': json.dumps(METADATA) 	# 发送元数据
	# **************************************
}

DATA = {
    "messages": [{"role": "user", "content": "What happens when you mix red & yellow?"}],
    "model": "mistralai/Mistral-7B-Instruct-v0.1"
}

response = requests.post(PORTKEY_GATEWAY_URL, headers=PORTKEY_HEADERS, json=DATA)

print(response.text)

""" CURL """
curl "https://api.portkey.ai/v1/chat/completions" \
  -H 'Content-Type: application/json' \
  -H 'Authorization: Bearer ANYSCALE_KEY' \
  -H 'x-portkey-api-key: PORTKEY_KEY' \
  -H 'x-portkey-provider: anyscale' \
  -H 'x-portkey-trace-id: TRACE_ID' \
  -H 'x-portkey-metadata: {"_environment": "production","_user": "userid123","_organisation": "orgid123","_prompt": "summarisationPrompt"}' \
  -d '{
    "model": "meta-llama/Llama-2-70b-chat-hf",
    "messages": [{"role": "user", "content": "Say 'Test'."}]
  }'

您的日志将在 Portkey 仪表板上显示如下：

2. 缓存、回退、负载均衡

• 回退：确保您的应用在主要服务失败时仍然能够正常运行。

• 负载均衡：高效地在多个模型之间分配传入请求。

• 语义缓存：通过智能缓存结果来降低成本和延迟。

通过保存 Configs（从 Portkey 仪表板 > Configs 标签）来切换这些功能。

如果我们想启用语义缓存 + 从 Llama2 回退到 Mistral，您的 Portkey 配置将如下所示：

{
  "cache": { "mode": "semantic" },
  "strategy": { "mode": "fallback" },
  "targets": [
    {
      "provider": "anyscale",
      "api_key": "...",
      "override_params": { "model": "meta-llama/Llama-2-7b-chat-hf" }
    },
    {
      "provider": "anyscale",
      "api_key": "...",
      "override_params": { "model": "mistralai/Mistral-7B-Instruct-v0.1" }
    }
  ]
}

现在，只需发送带有 x-portkey-config 头的配置 ID：

""" OPENAI PYTHON SDK """
import openai, json

PORTKEY_GATEWAY_URL = "https://api.portkey.ai/v1"

PORTKEY_HEADERS = {
	'Content-Type': 'application/json',
	'x-portkey-api-key': 'PORTKEY_API_KEY',
	# **************************************
	'x-portkey-config': 'CONFIG_ID'
	# **************************************
}

client = openai.OpenAI(base_url=PORTKEY_GATEWAY_URL, default_headers=PORTKEY_HEADERS)

response = client.chat.completions.create(
	model="mistralai/Mistral-7B-Instruct-v0.1",
	messages=[{"role": "user", "content": "Say this is a test"}]
)

print(response.choices[0].message.content)

""" OPENAI NODE SDK """
import OpenAI from 'openai';

const PORTKEY_GATEWAY_URL = "https://api.portkey.ai/v1"

const PORTKEY_HEADERS = {
	'Content-Type': 'application/json',
	'x-portkey-api-key': 'PORTKEY_API_KEY',
	// **************************************
	'x-portkey-config': 'CONFIG_ID'
	// **************************************
}

const openai = new OpenAI({baseURL:PORTKEY_GATEWAY_URL, defaultHeaders:PORTKEY_HEADERS});

async function main() {
  const chatCompletion = await openai.chat.completions.create({
    messages: [{ role: 'user', content: 'Say this is a test' }],
    model: 'mistralai/Mistral-7B-Instruct-v0.1',
  });
  console.log(chatCompletion.choices[0].message.content);
}

main();

""" REQUESTS LIBRARY """
import requests, json

PORTKEY_GATEWAY_URL = "https://api.portkey.ai/v1/chat/completions"

PORTKEY_HEADERS = {
	'Content-Type': 'application/json',
	'x-portkey-api-key': 'PORTKEY_API_KEY',
	# **************************************
	'x-portkey-config': 'CONFIG_ID'
	# **************************************
}

DATA = {"messages": [{"role": "user", "content": "What happens when you mix red & yellow?"}]}

response = requests.post(PORTKEY_GATEWAY_URL, headers=PORTKEY_HEADERS, json=DATA)

print(response.text)

""" CURL """
curl "https://api.portkey.ai/v1/chat/completions" \
  -H "Content-Type: application/json" \
  -H "x-portkey-api-key: PORTKEY_API_KEY" \
  -H "x-portkey-config: CONFIG_ID" \
  -d '{ "messages": [{"role": "user", "content": "Say 'Test'."}] }'

有关 Configs 和其他网关功能如负载均衡的更多信息，请 查看文档。

3. 收集反馈

从用户那里收集加权反馈并改进您的应用：

""" REQUESTS LIBRARY """
import requests
import json

PORTKEY_FEEDBACK_URL = "https://api.portkey.ai/v1/feedback" # Portkey 反馈端点

PORTKEY_HEADERS = {
	"x-portkey-api-key": "PORTKEY_API_KEY",
	"Content-Type": "application/json",
}

DATA = {
	"trace_id": "anyscale_portkey_test", # 在 Portkey 上，您可以将反馈附加到特定的 Trace ID
	"value": 1,
	"weight": 0.5
}

response = requests.post(PORTKEY_FEEDBACK_URL, headers=PORTKEY_HEADERS, data=json.dumps(DATA))

print(response.text)

""" CURL """
curl "https://api.portkey.ai/v1/feedback" \
  -H "x-portkey-api-key: PORTKEY_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "trace_id": "anyscale_portkey_test",
    "value": 1,
    "weight": 0.5
  }'

4. 持续微调

一旦您开始使用 Portkey 记录请求及其反馈，就可以非常轻松地 1️) 策划和创建微调数据，2) 安排微调任务，以及 3) 使用微调后的模型！

目前，微调功能仅对部分组织开放 - 请在 Portkey Discord 上申请访问权限，我们会尽快回复您。

结论

将 Portkey 与 Anyscale 集成可以帮助您从一开始就构建弹性的 LLM 应用程序。借助语义缓存、可观察性、负载均衡、反馈和后备等功能，您可以确保最佳性能和持续改进。

在这里阅读完整的 Portkey 文档。 | 联系 Portkey 团队。