Skip to content

VertexAI: migrate test cases to use updated set of golden files #8324

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 20, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 3 additions & 7 deletions packages/vertexai/src/methods/count-tokens.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,7 @@ describe('countTokens()', () => {
restore();
});
it('total tokens', async () => {
const mockResponse = getMockResponse(
'count-tokens-success-total-tokens.json'
);
const mockResponse = getMockResponse('unary-success-total-tokens.json');
const makeRequestStub = stub(request, 'makeRequest').resolves(
mockResponse as Response
);
Expand All @@ -70,7 +68,7 @@ describe('countTokens()', () => {
});
it('total tokens no billable characters', async () => {
const mockResponse = getMockResponse(
'count-tokens-success-no-billable-characters.json'
'unary-success-no-billable-characters.json'
);
const makeRequestStub = stub(request, 'makeRequest').resolves(
mockResponse as Response
Expand All @@ -94,9 +92,7 @@ describe('countTokens()', () => {
);
});
it('model not found', async () => {
const mockResponse = getMockResponse(
'count-tokens-failure-model-not-found.json'
);
const mockResponse = getMockResponse('unary-failure-model-not-found.json');
const mockFetch = stub(globalThis, 'fetch').resolves({
ok: false,
status: 404,
Expand Down
10 changes: 6 additions & 4 deletions packages/vertexai/src/methods/generate-content.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ describe('generateContent()', () => {
'model',
fakeRequestParams
);
expect(result.response.text()).to.include('Helena');
expect(result.response.text()).to.include('Mountain View, California');
expect(makeRequestStub).to.be.calledWith(
'model',
Task.GENERATE_CONTENT,
Expand Down Expand Up @@ -112,10 +112,12 @@ describe('generateContent()', () => {
'model',
fakeRequestParams
);
expect(result.response.text()).to.include('Quantum mechanics is');
expect(result.response.text()).to.include(
'Some information cited from an external source'
);
expect(
result.response.candidates?.[0].citationMetadata?.citations.length
).to.equal(1);
).to.equal(3);
expect(makeRequestStub).to.be.calledWith(
'model',
Task.GENERATE_CONTENT,
Expand Down Expand Up @@ -186,7 +188,7 @@ describe('generateContent()', () => {
);
});
it('unknown enum - should ignore', async () => {
const mockResponse = getMockResponse('unary-unknown-enum.json');
const mockResponse = getMockResponse('unary-success-unknown-enum.json');
const makeRequestStub = stub(request, 'makeRequest').resolves(
mockResponse as Response
);
Expand Down
4 changes: 1 addition & 3 deletions packages/vertexai/src/models/generative-model.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -264,9 +264,7 @@ describe('GenerativeModel', () => {
});
it('calls countTokens', async () => {
const genModel = new GenerativeModel(fakeVertexAI, { model: 'my-model' });
const mockResponse = getMockResponse(
'count-tokens-success-total-tokens.json'
);
const mockResponse = getMockResponse('unary-success-total-tokens.json');
const makeRequestStub = stub(request, 'makeRequest').resolves(
mockResponse as Response
);
Expand Down
4 changes: 3 additions & 1 deletion packages/vertexai/src/requests/stream-reader.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,9 @@ describe('processStream', () => {
}
});
it('unknown enum - should ignore', async () => {
const fakeResponse = getMockResponseStreaming('streaming-unknown-enum.txt');
const fakeResponse = getMockResponseStreaming(
'streaming-success-unknown-enum.txt'
);
const result = processStream(fakeResponse as Response);
const aggregatedResponse = await result.response;
expect(aggregatedResponse.text()).to.include('Cats');
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
data: {"candidates": [{"content": {"parts": [{"text": "No"}]},"finishReason": "SAFETY","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "HIGH"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}],"promptFeedback": {"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}}
data: {"candidates": [{"content": {"parts": [{"text": "<redacted>"}],"role": "model"},"finishReason": "SAFETY","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "HIGH"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}],"promptFeedback": {"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}}

Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
data: {"candidates": [{"content": {"parts": [{"text": "**Cats:**\n\n- **Physical Characteristics:**\n - Size: Cats come"}]},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}],"promptFeedback": {"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}}
data: {"candidates": [{"content": {"parts": [{"text": "**Cats:**\n\n- **Physical Characteristics:**\n - Size: Cats come"}],"role": "model"},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}],"promptFeedback": {"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}}

data: {"candidates": [{"content": {"parts": [{"text": " in a wide range of sizes, from small breeds like the Singapura to large breeds like the Maine Coon.\n - Fur: Cats have soft, furry coats"}]},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}]}
data: {"candidates": [{"content": {"parts": [{"text": " in a wide range of sizes, from small breeds like the Singapura to large breeds like the Maine Coon.\n - Fur: Cats have soft, furry coats"}],"role": "model"},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}]}

data: {"candidates": [{"content": {"parts": [{"text": " that can vary in length and texture depending on the breed.\n - Eyes: Cats have large, expressive eyes that can be various colors, including green, blue, yellow, and hazel.\n - Ears: Cats have pointed, erect ears that are sensitive to sound.\n - Tail: Cats have long"}]},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}]}
data: {"candidates": [{"content": {"parts": [{"text": " that can vary in length and texture depending on the breed.\n - Eyes: Cats have large, expressive eyes that can be various colors, including green, blue, yellow, and hazel.\n - Ears: Cats have pointed, erect ears that are sensitive to sound.\n - Tail: Cats have long"}],"role": "model"},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}]}

data: {"candidates": [{"content": {"parts": [{"text": ", flexible tails that they use for balance and communication.\n\n- **Behavior and Personality:**\n - Independent: Cats are often described as independent animals that enjoy spending time alone.\n - Affectionate: Despite their independent nature, cats can be very affectionate and form strong bonds with their owners.\n - Playful: Cats are naturally playful and enjoy engaging in activities such as chasing toys, climbing, and pouncing.\n - Curious: Cats are curious creatures that love to explore their surroundings.\n - Vocal: Cats communicate through a variety of vocalizations, including meows, purrs, hisses, and grow"}]},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}]}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
data: {"candidates": [{"content": {"parts": [{"text": "Cheyenne"}]},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}],"promptFeedback": {"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}}
data: {"candidates": [{"content": {"role": "model","parts": [{"text": "Cheyenne"}]},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE","probabilityScore": 0.02854415,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.052424565},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE","probabilityScore": 0.24926445,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.0996453},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE","probabilityScore": 0.087096825,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.043123372},{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE","probabilityScore": 0.14402841,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.086169556}]}],"promptFeedback": {"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]},"usageMetadata": {"promptTokenCount": 6,"candidatesTokenCount": 4,"totalTokenCount": 10}}

Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
data: {"candidates": [{"content": {"parts": [{"text": "1. **Definition:**\nQuantum mechanics is a fundamental theory in physics that provides"}],"role": "model"},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}],"promptFeedback": {"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}}
data: {"candidates": [{"content": {"parts": [{"text": "1. **Definition:**\nQuantum mechanics is a fundamental theory in physics that provides"}],"role": "model"},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}]}

data: {"candidates": [{"content": {"parts": [{"text": " the foundation for understanding the physical properties of nature at the scale of atoms and subatomic particles. It is based on the idea that energy, momentum, angular momentum"}],"role": "model"},"finishReason": "STOP","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}]}

Expand Down
Loading
Loading