Feature/prompt limit issue (#3) -> Prompt limit increase with split the prompt if exceeded number of tokens

Prompt limit increase with split the prompt if exceeded number of tokens

Author: Infracloud-harsh

.github/workflows/openai-review.yml

@@ -11,8 +11,7 @@ on:
     types: [created]
 
 concurrency:
-  group:
-    ${{ github.repository }}-${{ github.event.number || github.head_ref ||
+  group: ${{ github.repository }}-${{ github.event.number || github.head_ref ||
     github.sha }}-${{ github.workflow }}-${{ github.event_name ==
     'pull_request_review_comment' && 'pr_comment' || 'pr' }}
   cancel-in-progress: ${{ github.event_name != 'pull_request_review_comment' }}

README.md

@@ -43,14 +43,23 @@ configure the required environment variables, such as `GITHUB_TOKEN` and
 `OPENAI_API_KEY`. For more information on usage, examples, contributing, and
 FAQs, you can refer to the sections below.
 
-- [Overview](#overview)
-- [Professional Version of CodeRabbit](#professional-version-of-coderabbit)
-- [Reviewer Features](#reviewer-features)
-- [Install instructions](#install-instructions)
-- [Conversation with CodeRabbit](#conversation-with-coderabbit)
-- [Examples](#examples)
-- [Contribute](#contribute)
-- [FAQs](#faqs)
+- [AI-based PR reviewer and summarizer](#ai-based-pr-reviewer-and-summarizer)
+  - [Overview](#overview)
+  - [Reviewer Features:](#reviewer-features)
+  - [CodeRabbit Pro](#coderabbit-pro)
+  - [Install instructions](#install-instructions)
+      - [Environment variables](#environment-variables)
+    - [Models: `gpt-4` and `gpt-3.5-turbo`](#models-gpt-4-and-gpt-35-turbo)
+    - [Prompts \& Configuration](#prompts--configuration)
+  - [Conversation with CodeRabbit](#conversation-with-coderabbit)
+    - [Ignoring PRs](#ignoring-prs)
+  - [Examples](#examples)
+  - [Contribute](#contribute)
+    - [Developing](#developing)
+  - [FAQs](#faqs)
+    - [Review pull requests from forks](#review-pull-requests-from-forks)
+    - [Inspect the messages between OpenAI server](#inspect-the-messages-between-openai-server)
+    - [Disclaimer](#disclaimer)
 
 ## CodeRabbit Pro
 
@@ -209,12 +218,11 @@ Install the dependencies
 $ npm install
 ```
 
-Build the typescript and package it for distribution
-
 ```bash
 $ npm run build && npm run package
 ```
 
+
 ## FAQs
 
 ### Review pull requests from forks

__tests__/tokenizer.test.ts

@@ -0,0 +1,47 @@
+import {splitPrompt} from './../src/tokenizer' // Import your module with the splitPrompt function
+
+describe('splitPrompt function', () => {
+  it('should split a prompt into smaller pieces', async () => {
+    const maxTokens = 10 // Adjust this as needed
+    const prompt = 'This is a test prompt for splitting into smaller pieces.'
+
+    const result = await splitPrompt(maxTokens, prompt)
+
+    // Calculate the expected output based on the maxTokens value
+    const expectedOutput = [
+      'This is a',
+      'test',
+      'prompt for',
+      'splitting',
+      'into',
+      'smaller',
+      'pieces.'
+    ]
+
+    expect(result).toEqual(expectedOutput)
+  })
+
+  it('should handle a prompt smaller than maxTokens', async () => {
+    const maxTokens = 100 // A large value
+    const prompt = 'A very short prompt.'
+
+    const result = await splitPrompt(maxTokens, prompt)
+
+    // The prompt is already smaller than maxTokens, so it should return an array with the entire prompt.
+    const expectedOutput = 'A very short prompt.'
+
+    expect(result).toEqual(expectedOutput)
+  })
+
+  it('should handle an empty prompt', async () => {
+    const maxTokens = 10
+    const prompt = ''
+
+    const result = await splitPrompt(maxTokens, prompt)
+
+    // An empty prompt should result in an empty array.
+    const expectedOutput: string[] | string = ''
+
+    expect(result).toEqual(expectedOutput)
+  })
+})