diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 203238b..3da178e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -30,7 +30,7 @@ jobs:
       with:
         version: "*"
         token: ${{ secrets.GITHUB_TOKEN }}
-        allow-external-github-orgs: true
+        # allow-external-github-orgs: true
     - run: foreman --version
     - run: rojo --version
 
@@ -55,6 +55,6 @@ jobs:
         version: "*"
         token: ${{ secrets.GITHUB_TOKEN }}
         working-directory: tests
-        allow-external-github-orgs: true
+        # allow-external-github-orgs: true
     - run: foreman --version
     - run: selene --version
diff --git a/__tests__/hello.test.ts b/__tests__/configFile.test.ts
similarity index 78%
rename from __tests__/hello.test.ts
rename to __tests__/configFile.test.ts
index 597fa4b..89fd09e 100644
--- a/__tests__/hello.test.ts
+++ b/__tests__/configFile.test.ts
@@ -1,6 +1,5 @@
-// who needs tests?
 import configFile from "../src/configFile";
-import {parse} from "toml";
+import { parse } from "toml";
 
 test("get off my back, Jest", () => {
   expect(5).toEqual(5);
@@ -14,7 +13,7 @@ test("checkSameOrgToolSpec same org", () => {
     tool3 = { source = "org1/tool3", version = "1.0.0" }\n
   `;
   let manifestContent = parse(config);
-  expect(configFile.checkSameOrgToolSpecs(manifestContent)).toEqual(true);
+  expect(configFile.checkSameOrgToolSpecs(manifestContent, "org1")).toEqual(true);
 });
 
 test("checkSameOrgToolSpec different org", () => {
@@ -25,5 +24,5 @@ test("checkSameOrgToolSpec different org", () => {
     tool3 = { source = "org1/tool3", version = "1.0.0" }\n
   `;
   let manifestContent = parse(config);
-  expect(configFile.checkSameOrgToolSpecs(manifestContent)).toEqual(false);
+  expect(configFile.checkSameOrgToolSpecs(manifestContent, "org1")).toEqual(false);
 });
diff --git a/src/configFile.ts b/src/configFile.ts
index 463d498..a0fdcb0 100644
--- a/src/configFile.ts
+++ b/src/configFile.ts
@@ -16,13 +16,12 @@ interface foremanTool {
 
 const MANIFEST = "foreman.toml";
 
-function checkSameOrgToolSpecs(manifestContent: foremanConfig): boolean {
+function checkSameOrgToolSpecs(manifestContent: foremanConfig, org: string): boolean {
   const tools = manifestContent.tools;
   if (tools == null) {
     throw new Error("Tools section in Foreman config not found");
   }
 
-  const orgs: string[] = [];
   for (const tool_name in tools) {
     const tool_spec = tools[tool_name];
     let source = tool_spec["source"];
@@ -34,35 +33,35 @@ function checkSameOrgToolSpecs(manifestContent: foremanConfig): boolean {
     }
 
     const source_array = source.split("/");
-    const org = source_array[0];
+    const tool_org = source_array[0];
 
-    if (org == null) {
+    if (tool_org == null) {
       throw new Error(
         `Org not found in tool spec definition for: ${tool_name}`
       );
     }
-    orgs.push(org.toLowerCase());
-  }
-  if (orgs.length == 0) {
-    return true;
+
+    if (tool_org != org) {
+      return false
+    }
   }
-  return orgs.every(val => val === orgs[0]);
+  return true
 }
 
-async function checkSameOrgInConfig(): Promise<void> {
+async function checkSameOrgInConfig(org: string): Promise<void> {
   const manifestPath = await findUp(MANIFEST);
   if (manifestPath == null) {
-    throw new Error("Foreman config file could not be found");
+    throw new Error("setup-foreman could not find Foreman config file");
   }
 
   await readFile(manifestPath, "utf8", (err, data) => {
     if (err) {
-      throw new Error("Could not read Foreman config file");
+      throw new Error("setup-foreman Could not read Foreman config file");
     }
     const manifestContent = parse(data);
-    const sameGithubOrgSource = checkSameOrgToolSpecs(manifestContent);
+    const sameGithubOrgSource = checkSameOrgToolSpecs(manifestContent, org);
     if (sameGithubOrgSource == false) {
-      throw new Error("Not all GitHub orgs are the same");
+      throw new Error(`All GitHub orgs in Foreman config must match the org setup-foreman runs in: ${org}. To disable this check, set the \"allow-external-github-orgs\" option to true.`);
     }
   });
 }
diff --git a/src/foreman.ts b/src/foreman.ts
index 718a5c9..6120e2e 100644
--- a/src/foreman.ts
+++ b/src/foreman.ts
@@ -1,6 +1,6 @@
-import {addPath} from "@actions/core";
-import {exec} from "@actions/exec";
-import {GitHub} from "@actions/github";
+import { addPath } from "@actions/core";
+import { exec } from "@actions/exec";
+import { GitHub } from "@actions/github";
 import semver from "semver";
 import os from "os";
 
diff --git a/src/main.ts b/src/main.ts
index 0c1f514..f0cabb9 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -1,8 +1,8 @@
-import {getInput, debug, addPath, setFailed} from "@actions/core";
-import {downloadTool, extractZip} from "@actions/tool-cache";
-import {GitHub} from "@actions/github";
-import {resolve} from "path";
-import {exec} from "@actions/exec";
+import { getInput, debug, addPath, setFailed } from "@actions/core";
+import { downloadTool, extractZip } from "@actions/tool-cache";
+import { GitHub, context } from "@actions/github";
+import { resolve } from "path";
+import { exec } from "@actions/exec";
 import configFile from "./configFile";
 import foreman from "./foreman";
 
@@ -16,12 +16,23 @@ async function run(): Promise<void> {
     ).toLowerCase();
 
     if (allowExternalGithubOrgs != "true") {
-      configFile.checkSameOrgInConfig();
+      let repo = context.payload.repository;
+      if (repo == null) {
+        throw new Error(
+          `Could not find repository`
+        )
+      }
+      let org = repo.owner.name;
+      if (org == null) {
+        throw new Error(
+          `Could not find owner of the repository`
+        )
+      }
+      configFile.checkSameOrgInConfig(org);
     }
 
     const octokit = new GitHub(githubToken);
     const releases = await foreman.getReleases(octokit);
-
     debug("Choosing release from GitHub API");
 
     const release = foreman.chooseRelease(versionReq, releases);