From 59c28dc17e75b234be59f34cab135be550bfa17d Mon Sep 17 00:00:00 2001 From: Anders Starcke Henriksen Date: Tue, 1 Aug 2023 16:17:32 +0200 Subject: [PATCH] Draft PR showcasing how to use filtered queries in the extension. --- .../auto-model-codeml-queries.ts | 12 ++- .../data-extensions-editor-view.ts | 80 ++++++++++++++++++- 2 files changed, 89 insertions(+), 3 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model-codeml-queries.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model-codeml-queries.ts index ca42902bab1..bc1b1331607 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/auto-model-codeml-queries.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model-codeml-queries.ts @@ -26,6 +26,7 @@ type AutoModelQueryOptions = { databaseItem: DatabaseItem; qlpack: QlPacksForLanguage; sourceInfo: SourceInfo | undefined; + additionalPacks: string[]; extensionPacks: string[]; queryStorageDir: string; @@ -52,6 +53,7 @@ async function runAutoModelQuery({ databaseItem, qlpack, sourceInfo, + additionalPacks, extensionPacks, queryStorageDir, progress, @@ -99,7 +101,7 @@ async function runAutoModelQuery({ quickEvalCountOnly: false, }, false, - getOnDiskWorkspaceFolders(), + additionalPacks, extensionPacks, queryStorageDir, undefined, @@ -153,6 +155,7 @@ type AutoModelQueriesOptions = { queryStorageDir: string; progress: ProgressCallback; + extraExtensionPacks?: string[]; }; export type AutoModelQueriesResult = { @@ -166,6 +169,7 @@ export async function runAutoModelQueries({ databaseItem, queryStorageDir, progress, + extraExtensionPacks = [], }: AutoModelQueriesOptions): Promise { // maxStep for this part is 1500 const maxStep = 1500; @@ -189,7 +193,10 @@ export async function runAutoModelQueries({ sourceLocationPrefix, }; - const additionalPacks = getOnDiskWorkspaceFolders(); + const additionalPacks = [ + ...getOnDiskWorkspaceFolders(), + ...extraExtensionPacks, + ]; const extensionPacks = Object.keys( await cliServer.resolveQlpacks(additionalPacks, true), ); @@ -208,6 +215,7 @@ export async function runAutoModelQueries({ databaseItem, qlpack, sourceInfo, + additionalPacks, extensionPacks, queryStorageDir, progress: (update) => { diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index 888893c94ca..af148406aa3 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -57,8 +57,13 @@ import { pickExtensionPack } from "./extension-pack-picker"; import { getLanguageDisplayName } from "../common/query-language"; import { runAutoModelQueries } from "./auto-model-codeml-queries"; import { createAutoModelV2Request } from "./auto-model-v2"; -import { load as loadYaml } from "js-yaml"; +import { load as loadYaml, dump as dumpYaml } from "js-yaml"; import { loadDataExtensionYaml } from "./yaml"; +import { extLogger } from "../common/logging/vscode"; +import { dir } from "tmp-promise"; +import { writeFile, outputFile } from "fs-extra"; +import { autoPickExtensionsDirectory } from "./extensions-workspace-folder"; +import { sign } from "crypto"; export class DataExtensionsEditorView extends AbstractWebview< ToDataExtensionsEditorMessage, @@ -380,6 +385,46 @@ export class DataExtensionsEditorView extends AbstractWebview< let predictedModeledMethods: Record; if (useLlmGenerationV2()) { + // Generate a qlpack with a filter that only includes the usages we want to model. + const packDir = (await dir({ unsafeCleanup: true })).path; + + const syntheticConfigPack = { + name: "codeql/automodel-filter", + version: "0.0.0", + library: true, + extensionTargets: { + [`codeql/${this.databaseItem.language}-all`]: "*", + }, + dataExtensions: ["filter.yml"], + }; + + const qlpackFile = join(packDir, "codeql-pack.yml"); + await outputFile(qlpackFile, dumpYaml(syntheticConfigPack), "utf8"); + + // TODO: this filter is just static as an example. We want to generate this from the usages. + const filter = { + extensions: [ + { + addsTo: { + pack: `codeql/${this.databaseItem.language}-queries`, + extensible: "automodelCandidateFilter", + }, + data: [ + ["org.sql2o", "Sql2o", "open", "()"], + [ + "org.springframework.boot", + "SpringApplication", + "run", + "(Class,String[])", + ], + ], + }, + ], + }; + + const filterFile = join(packDir, "filter.yml"); + await writeFile(filterFile, dumpYaml(filter), "utf8"); + const usages = await runAutoModelQueries({ mode: this.mode, cliServer: this.cliServer, @@ -387,6 +432,7 @@ export class DataExtensionsEditorView extends AbstractWebview< queryStorageDir: this.queryStorageDir, databaseItem: this.databaseItem, progress: (update) => progress({ ...update, maxStep }), + extraExtensionPacks: [packDir], }); if (!usages) { return; @@ -398,6 +444,38 @@ export class DataExtensionsEditorView extends AbstractWebview< message: "Creating request", }); + // TODO: TEMP LOGGING CODE - START + const results = usages.candidates.runs[0].results; + void extLogger.log(`CANDIDATES:`); + results?.forEach((result) => { + const pckage = + result.relatedLocations?.[1].physicalLocation?.artifactLocation?.uri?.substring( + 6, + ); + const tp = + result.relatedLocations?.[2].physicalLocation?.artifactLocation?.uri?.substring( + 6, + ); + const method = + result.relatedLocations?.[4].physicalLocation?.artifactLocation?.uri?.substring( + 6, + ); + let signature = + result.relatedLocations?.[5].physicalLocation?.artifactLocation?.uri?.substring( + 6, + ); + signature = signature && decodeURI(signature); + let input = + result.relatedLocations?.[6].physicalLocation?.artifactLocation?.uri?.substring( + 6, + ); + input = input && decodeURI(input); + void extLogger.log( + `${pckage}.${tp}.${method}${signature} @ ${input}`, + ); + }); + // TODO: TEMP LOGGING CODE - END + const request = await createAutoModelV2Request(this.mode, usages); progress({