diff --git a/.azure-pipelines/rc.yml b/.azure-pipelines/rc.yml
index 1f7bd599..07ab4763 100644
--- a/.azure-pipelines/rc.yml
+++ b/.azure-pipelines/rc.yml
@@ -88,6 +88,13 @@ extends:
                 displayName: Replace AI Key
                 inputs:
                   script: npx json@9.0.6 -I -f package.json -e "this.aiKey=\"%AI_KEY%\""
+              - task: PowerShell@2
+                displayName: Update package.json for stable
+                inputs:
+                  targetType: inline
+                  script: |-
+                    node ./scripts/prepare-stable-build.js
+                    Move-Item -Path "./package.stable.json" -Destination "./package.json" -Force
               - task: CmdLine@2
                 displayName: vsce package
                 inputs:
diff --git a/package-lock.json b/package-lock.json
index 1dc95bfe..4078800b 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,6 +1,6 @@
 {
   "name": "vscode-java-dependency",
-  "version": "0.27.0",
+  "version": "0.27.1",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
diff --git a/package.json b/package.json
index 25e76efd..b43d2647 100644
--- a/package.json
+++ b/package.json
@@ -2,7 +2,7 @@
   "name": "vscode-java-dependency",
   "displayName": "Project Manager for Java",
   "description": "%description%",
-  "version": "0.27.0",
+  "version": "0.27.1",
   "publisher": "vscjava",
   "preview": false,
   "aiKey": "5c642b22-e845-4400-badb-3f8509a70777",
diff --git a/scripts/prepare-nightly-build.js b/scripts/prepare-nightly-build.js
index 7e40038b..a41983f3 100644
--- a/scripts/prepare-nightly-build.js
+++ b/scripts/prepare-nightly-build.js
@@ -2,12 +2,15 @@ const fs = require("fs");
 
 const json = JSON.parse(fs.readFileSync("./package.json").toString());
 const stableVersion = json.version.match(/(\d+)\.(\d+)\.(\d+)/);
+if (!stableVersion) {
+    throw new Error(`Invalid stable version: ${json.version}`);
+}
 const major = stableVersion[1];
 const minor = stableVersion[2];
 
 function prependZero(number) {
     if (number > 99) {
-        throw "Unexpected value to prepend with zero";
+        throw new Error("Unexpected value to prepend with zero");
     }
     return `${number < 10 ? "0" : ""}${number}`;
 }
@@ -16,10 +19,11 @@ const date = new Date();
 const month = date.getMonth() + 1;
 const day = date.getDate();
 const hours = date.getHours();
-patch = `${date.getFullYear()}${prependZero(month)}${prependZero(day)}${prependZero(hours)}`;
+const patch = `${date.getFullYear()}${prependZero(month)}${prependZero(day)}${prependZero(hours)}`;
 
 const insiderPackageJson = Object.assign(json, {
     version: `${major}.${minor}.${patch}`,
+    preview: true,
 });
 
-fs.writeFileSync("./package.insiders.json", JSON.stringify(insiderPackageJson));
\ No newline at end of file
+fs.writeFileSync("./package.insiders.json", `${JSON.stringify(insiderPackageJson, null, 2)}\n`);
\ No newline at end of file
diff --git a/scripts/prepare-stable-build.js b/scripts/prepare-stable-build.js
new file mode 100644
index 00000000..2596a598
--- /dev/null
+++ b/scripts/prepare-stable-build.js
@@ -0,0 +1,17 @@
+const fs = require("fs");
+
+const packageJsonPath = "./package.json";
+const packageJson = JSON.parse(fs.readFileSync(packageJsonPath).toString());
+packageJson.preview = false;
+
+if (packageJson.contributes) {
+    delete packageJson.contributes.languageModelTools;
+    delete packageJson.contributes.chatSkills;
+    delete packageJson.contributes.chatInstructions;
+
+    if (packageJson.contributes.configuration && packageJson.contributes.configuration.properties) {
+        delete packageJson.contributes.configuration.properties["vscode-java-dependency.enableLspTools"];
+    }
+}
+
+fs.writeFileSync("./package.stable.json", `${JSON.stringify(packageJson, null, 2)}\n`);
diff --git a/src/extension.ts b/src/extension.ts
index aa915423..4c5c9637 100644
--- a/src/extension.ts
+++ b/src/extension.ts
@@ -2,8 +2,10 @@
 // Licensed under the MIT license.
 
 import * as path from "path";
-import { commands, Diagnostic, Extension, ExtensionContext, extensions, languages,
-    Range, tasks, TextDocument, TextEditor, Uri, window, workspace } from "vscode";
+import {
+    commands, Diagnostic, Extension, ExtensionContext, extensions, languages,
+    Range, tasks, TextDocument, TextEditor, Uri, window, workspace
+} from "vscode";
 import { dispose as disposeTelemetryWrapper, initializeFromJsonFile, instrumentOperation, instrumentOperationAsVsCodeCommand, sendInfo } from "vscode-extension-telemetry-wrapper";
 import { Commands, contextManager } from "../extension.bundle";
 import { BuildTaskProvider } from "./tasks/build/buildTaskProvider";
@@ -87,9 +89,13 @@ async function activateExtension(_operationId: string, context: ExtensionContext
     setContextForDeprecatedTasks();
 
     // Register Copilot context providers after Java Language Server is ready
+    const isPrereleaseBuild = context.extension.packageJSON.preview === true;
+    if (!isPrereleaseBuild) {
+        return;
+    }
     languageServerApiManager.ready().then((isReady) => {
         const config = workspace.getConfiguration("vscode-java-dependency");
-        const isSettingEnabled = config.get<boolean>("enableLspTools", true);
+        const isSettingEnabled = config.get<boolean>("enableLspTools", false);
         if (isReady && isSettingEnabled) {
             registerJavaContextTools(context);
         }