provide option to review simple changes (#232)

Author: harjotgill

README.md

@@ -5,13 +5,47 @@
 ## Overview
 
 This [OpenAI ChatGPT-based](https://platform.openai.com/docs/guides/chat) GitHub
-Action provides a summary, release notes and review of pull requests. The
-prompts have been tuned for a concise response. To prevent excessive
-notifications, this action can be configured to skip adding review comments when
-the changes look good for the most part.
-
-In addition, this action can also reply to the user comments made on the review
-by this action.
+Action provides a summary, release notes and review of pull requests. The unique
+features of this action are:
+
+- Unlike other approaches that provide a simple summary and/or conversation,
+  this action reviews the changes line by line and provides code change
+  suggestions that can be directly committed from the GitHub UI. The prompts
+  have been tuned carefully to comment on exact lines within changed hunks of
+  code.
+- Continuous, yet incremental, reviews on each commit with a pull request. This
+  is unlike other approaches that provide a one-time review on the entire pull
+  request when requested by the user.
+- Incremental reviews save on OpenAI costs while also reducing noise. Changed
+  files are tracked between commits and the base of the pull request
+- The action is designed to be used with a "light" summarization model (e.g.
+  `gpt-3.5-turbo`) and a "heavy" review model (e.g. `gpt-4`). This allows for a
+  cheaper and faster summarization process and a more accurate review process.
+  **For best results, setting `gpt-4` as the "heavy" model instead of the
+  `gpt-3.5-turbo` (default) is highly recommended.**
+- This action supports a conversation with the bot in the context of lines of
+  code or entire files. Useful for providing further context to the bot for its
+  next review, to generate test cases, to reduce complexity of the code and so
+  on.
+- By default, the action is configured to skip more in-depth review when the
+  changes are simple (e.g. typo fixes). This is based on the triage done during
+  the summarization stage. This feature can be disabled by setting
+  `review_simple_changes` to `true`.
+- By default, the action is configured to skip adding review comments when the
+  changes look good for the most part. This feature can be disabled by setting
+  `review_comment_lgtm` to `true`.
+- You can tailor the following prompts:
+  - `system_message`: Defines the objective and the personality of the bot. You
+    can change this prompt to focus on or ignore certain aspects of the review
+    process, e.g. documentation, code quality, etc. Furthermore, you can even
+    change the bot to do marketing material review instead of code review.
+  - `summarize`: Summarizes the pull request into a table of changes etc.
+  - `summarize_release_notes`: Summarize the changes in the pull request for
+    release notes purposes.
+- You can altogether skip the reviews, setting the `summary_only` to `true`. But
+  that defeats the main purpose of this action. Other tools such as GitHub's
+  [Copliot for Pull Requests](https://githubnext.com/projects/copilot-for-pull-requests)
+  may be a cheaper and good enough alternative in that case.
 
 NOTES:
 
@@ -58,6 +92,7 @@ jobs:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         with:
           debug: false
+          review_simple_changes: false
           review_comment_lgtm: false
 ```
 
@@ -101,7 +136,9 @@ To ignore a PR, add the following keyword in the PR description:
 - `OPENAI_API_KEY`: use this to authenticate with OpenAI API. You can get one
   [here](https://platform.openai.com/account/api-keys). Please add this key to
   your GitHub Action secrets.
-- `OPENAI_API_ORG`: (optional) use this to use the specified organisation with OpenAI API if you have multiple. Please add this key to your GitHub Action secrets.
+- `OPENAI_API_ORG`: (optional) use this to use the specified organisation with
+  OpenAI API if you have multiple. Please add this key to your GitHub Action
+  secrets.
 
 ### Models: `gpt-4` and `gpt-3.5-turbo`
 
@@ -217,6 +254,7 @@ jobs:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         with:
           debug: false
+          review_simple_changes: false
           review_comment_lgtm: false
 ```
 

action.yml

@@ -15,6 +15,10 @@ inputs:
       'Max files to summarize and review. Less than or equal to 0 means no
       limit.'
     default: '150'
+  review_simple_changes:
+    required: false
+    description: 'Review even when the changes are simple'
+    default: 'false'
   review_comment_lgtm:
     required: false
     description: 'Leave comments even if the patch is LGTM'

dist/index.js

@@ -16,6 +16,7 @@ async function run(): Promise<void> {
     getBooleanInput('debug'),
     getBooleanInput('summary_only'),
     getInput('max_files'),
+    getBooleanInput('review_simple_changes'),
     getBooleanInput('review_comment_lgtm'),
     getMultilineInput('path_filters'),
     getInput('system_message'),

src/main.ts

@@ -6,6 +6,7 @@ export class Options {
   debug: boolean
   summaryOnly: boolean
   maxFiles: number
+  reviewSimpleChanges: boolean
   reviewCommentLGTM: boolean
   pathFilters: PathFilter
   systemMessage: string
@@ -22,6 +23,7 @@ export class Options {
     debug: boolean,
     summaryOnly: boolean,
     maxFiles = '0',
+    reviewSimpleChanges = false,
     reviewCommentLGTM = false,
     pathFilters: string[] | null = null,
     systemMessage = '',
@@ -35,6 +37,7 @@ export class Options {
     this.debug = debug
     this.summaryOnly = summaryOnly
     this.maxFiles = parseInt(maxFiles)
+    this.reviewSimpleChanges = reviewSimpleChanges
     this.reviewCommentLGTM = reviewCommentLGTM
     this.pathFilters = new PathFilter(pathFilters)
     this.systemMessage = systemMessage
@@ -53,6 +56,7 @@ export class Options {
     info(`debug: ${this.debug}`)
     info(`summary_only: ${this.summaryOnly}`)
     info(`max_files: ${this.maxFiles}`)
+    info(`review_simple_changes: ${this.reviewSimpleChanges}`)
     info(`review_comment_lgtm: ${this.reviewCommentLGTM}`)
     info(`path_filters: ${this.pathFilters}`)
     info(`system_message: ${this.systemMessage}`)

src/options.ts

@@ -23,8 +23,8 @@ $file_diff
 \`\`\`
 
 I would like you to summarize the diff within 50 words.
-
-Below the summary, I would also like you to triage the diff as \`NEEDS_REVIEW\` or 
+`
+  triageFileDiff = `Below the summary, I would also like you to triage the diff as \`NEEDS_REVIEW\` or 
 \`APPROVED\` based on the following criteria:
 
 - If the diff involves any modifications to the logic or functionality, even if they 
@@ -248,8 +248,15 @@ $patches
     this.summarizeReleaseNotes = summarizeReleaseNotes
   }
 
-  renderSummarizeFileDiff(inputs: Inputs): string {
-    return inputs.render(this.summarizeFileDiff)
+  renderSummarizeFileDiff(
+    inputs: Inputs,
+    reviewSimpleChanges: boolean
+  ): string {
+    let prompt = this.summarizeFileDiff
+    if (reviewSimpleChanges === false) {
+      prompt += this.triageFileDiff
+    }
+    return inputs.render(prompt)
   }
 
   renderSummarizeChangesets(inputs: Inputs): string {

src/prompts.ts

@@ -261,7 +261,9 @@ ${hunks.oldHunk}
     ins.filename = filename
 
     // render prompt based on inputs so far
-    let tokens = getTokenCount(prompts.renderSummarizeFileDiff(ins))
+    let tokens = getTokenCount(
+      prompts.renderSummarizeFileDiff(ins, options.reviewSimpleChanges)
+    )
 
     const diffTokens = getTokenCount(fileDiff)
     if (tokens + diffTokens > options.lightTokenLimits.requestTokens) {
@@ -291,7 +293,7 @@ ${hunks.oldHunk}
     // summarize content
     try {
       const [summarizeResp] = await lightBot.chat(
-        prompts.renderSummarizeFileDiff(ins),
+        prompts.renderSummarizeFileDiff(ins, options.reviewSimpleChanges),
         {}
       )
 
@@ -300,23 +302,24 @@ ${hunks.oldHunk}
         summariesFailed.push(`${filename} (nothing obtained from openai)`)
         return null
       } else {
-        // parse the comment to look for triage classification
-        // Format is : [TRIAGE]: <NEEDS_REVIEW or APPROVED>
-        // if the change needs review return true, else false
-        const triageRegex = /\[TRIAGE\]:\s*(NEEDS_REVIEW|APPROVED)/
-        const triageMatch = summarizeResp.match(triageRegex)
-
-        if (triageMatch != null) {
-          const triage = triageMatch[1]
-          const needsReview = triage === 'NEEDS_REVIEW'
-
-          // remove this line from the comment
-          const summary = summarizeResp.replace(triageRegex, '').trim()
-          info(`filename: ${filename}, triage: ${triage}`)
-          return [filename, summary, needsReview]
-        } else {
-          return [filename, summarizeResp, true]
+        if (options.reviewSimpleChanges === false) {
+          // parse the comment to look for triage classification
+          // Format is : [TRIAGE]: <NEEDS_REVIEW or APPROVED>
+          // if the change needs review return true, else false
+          const triageRegex = /\[TRIAGE\]:\s*(NEEDS_REVIEW|APPROVED)/
+          const triageMatch = summarizeResp.match(triageRegex)
+
+          if (triageMatch != null) {
+            const triage = triageMatch[1]
+            const needsReview = triage === 'NEEDS_REVIEW'
+
+            // remove this line from the comment
+            const summary = summarizeResp.replace(triageRegex, '').trim()
+            info(`filename: ${filename}, triage: ${triage}`)
+            return [filename, summary, needsReview]
+          }
         }
+        return [filename, summarizeResp, true]
       }
     } catch (e: any) {
       warning(`summarize: error from openai: ${e as string}`)
@@ -653,14 +656,19 @@ ${commentChain}
     await Promise.all(reviewPromises)
 
     summarizeComment += `
+---
+In the recent run, only the files that changed from the \`base\` of the PR and between \`${highestReviewedCommitId}\` and \`${
+      context.payload.pull_request.head.sha
+    }\` commits were reviewed.
+
 ${
   reviewsFailed.length > 0
     ? `<details>
-<summary>Files not reviewed due to errors in this run (${
+<summary>Files not reviewed due to errors in the recent run (${
         reviewsFailed.length
       })</summary>
 
-### Failed to review
+### Failed to review in the last run
 
 * ${reviewsFailed.join('\n* ')}
 
@@ -676,7 +684,7 @@ ${
         reviewsSkipped.length
       })</summary>
 
-### Skipped review
+### Skipped review in the recent run
 
 * ${reviewsSkipped.join('\n* ')}