Skip to content

Commit

Permalink
Handle many asset types selected
Browse files Browse the repository at this point in the history 
(cherry picked from commit 8c697d2)
  • Loading branch information
Fried Hoeben committed Oct 29, 2024
1 parent 41eee64 commit ad001f8
Show file tree
Hide file tree
Showing 4 changed files with 213 additions and 51 deletions.
79 changes: 63 additions & 16 deletions lansweeper/aws/integration-lambda/lansweeper_client.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -278,36 +278,83 @@ class LansweeperClient {
}

getFilters(withIP, assetCutOffDate, installationId, assetTypes = null) {
let conditions = '';
const conjunctions = [];

if (assetTypes) {
let assetTypesRegEx = this.helper.arrayToRegExValue(assetTypes);
conditions = `{operator: REGEXP, path: "assetBasicInfo.type", value: "${assetTypesRegEx}"}`;
const conditions =
this.getAssetTypeRegExs(assetTypes)
.map((regEx) => `{ operator: REGEXP, path: "assetBasicInfo.type", value: "${regEx}" }`);
const assetTypesConjunction = this.createConjunction("OR", ...conditions);
conjunctions.push(assetTypesConjunction);
} else {
if (withIP !== undefined) {
conditions = `{operator: EXISTS, path: "assetBasicInfo.ipAddress", value: "${withIP}"}`;
const ipConjunction = this.createConjunction("AND",
`{ operator: EXISTS, path: "assetBasicInfo.ipAddress", value: "${withIP}" }`);
conjunctions.push(ipConjunction);
}
}

const lastSeenConjunction = this.createConjunction("OR",
`{ operator: GREATER_THAN, path: "assetBasicInfo.lastSeen", value: "${assetCutOffDate.toISOString()}" }`,
'{ operator: EXISTS, path: "assetBasicInfo.lastSeen", value: "false" }',
);
conjunctions.push(lastSeenConjunction);

if (installationId !== undefined) {
// Here using the `installationId` in path results in an error response
conditions = `${conditions}\n{operator: EQUAL, path: "installationId", value: "${installationId}"}`;
const installationConjunction = this.createConjunction("AND",
`{ operator: EQUAL, path: "installationId", value: "${installationId}" }`);
conjunctions.push(installationConjunction);
}

return `{conjunction: OR, groups: [
{ conditions: [
${conditions}
{ operator: GREATER_THAN, path: "assetBasicInfo.lastSeen", value: "${assetCutOffDate.toISOString()}" }
]}
{ conditions: [
${conditions}
{ operator: EXISTS, path: "assetBasicInfo.lastSeen", value: "false" }
]}
]}`;
return `
{ conjunction: AND
groups: [ ${conjunctions.join(",\n")} ]
}`;
}

getAssetTypeRegExs(assetTypes) {
const [firstRegEx, remainingTypes] = this.firstRegexValue(assetTypes);
let regExs = [firstRegEx];
if (remainingTypes.length > 0) {
regExs.push(...this.getAssetTypeRegExs(remainingTypes));
}
return regExs;
}

createConjunction(operator, ...conditions) {
return `{ conjunction: ${operator}\n conditions: [ ${conditions.join(",\n")} ] }`;
}

firstRegexValue(assetTypes) {
let index = 0;
let currentRegEx = '';
let currentLength = -1;
let currentArray = [];
while (index < assetTypes.length && currentLength < LansweeperClient.maxFilterFieldLength) {
let nextType = assetTypes[index];
let nextLength = nextType.length;
if (nextLength > LansweeperClient.maxFilterFieldLength) {
console.error(`AssetType too long (${nextLength}): ${nextType}`);
} else {
const nextArray = [...currentArray, nextType];
const nextRegEx = this.helper.arrayToRegExValue(nextArray);
if (nextRegEx.length <= LansweeperClient.maxFilterFieldLength) {
currentArray = nextArray;
currentRegEx = nextRegEx;
currentLength = nextRegEx.length;
} else {
break;
}
}
index++;
}
const remainingArray = assetTypes.slice(index, assetTypes.length);
return [currentRegEx, remainingArray];
}
}

LansweeperClient.pageSize = parseInt(process.env.LANSWEEPER_PAGE_SIZE, 10) || 100;
LansweeperClient.maxFilterFieldLength = parseInt(process.env.LANSWEEPER_MAX_FILTER_FIELD_LENGTH, 10) || 100;
LansweeperClient.topLevelFields = '_id key url';
LansweeperClient.basicInfoFields = 'name type description ipAddress firstSeen lastSeen lastChanged userName userDomain';
LansweeperClient.assetCustomFields = 'model manufacturer stateName purchaseDate warrantyDate serialNumber sku';
Expand Down
4 changes: 4 additions & 0 deletions lansweeper/aws/integration-lambda/lansweeper_integration.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,10 @@ class LansweeperIntegration {
if (allAssetTypes.error) {
return allAssetTypes;
} else {
const tooLongAssetTypes = allAssetTypes.filter(t => t.length > LansweeperClient.maxFilterFieldLength);
if (tooLongAssetTypes.length > 0) {
console.warn(`Encountered ${tooLongAssetTypes.length} too long asset types: [${tooLongAssetTypes.join(',')}]`)
}
const allLower = allAssetTypes.map(t => t.toLowerCase());
const configLower = configAssetTypes.map(t => t.toLowerCase());
const assetTypes = configLower.filter(at => allLower.includes(at));
Expand Down
100 changes: 100 additions & 0 deletions lansweeper/aws/integration-lambda/tests/lansweeper_client.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,4 +55,104 @@ describe('retrieving installations with error for one site', () => {
expect(result).toEqual([{id: "a", name: "install A for b", siteId: "b"}, {id: "b", name: "install B for b", siteId: "b"}]);
});
});

describe('getFilters', () => {
const checkFilter = async (assetTypes, expectedFilter) => {
const client = createClient();

const result = await client.getFilters(undefined, new Date(2024, 9, 2), "e3", assetTypes);
if (cleanFilter(result) !== cleanFilter(expectedFilter)) {
expect(result).toEqual(expectedFilter);
}
}

const cleanFilter = (query) => query.replaceAll(/\s+/g, ' ').trim()

describe('asset filtering', () => {
it('no asset filtered', async () => {
const expectedQuery = `
{ conjunction: AND
groups: [
{ conjunction: OR
conditions: [
{ operator: GREATER_THAN, path: "assetBasicInfo.lastSeen", value: "2024-10-01T22:00:00.000Z" },
{ operator: EXISTS, path: "assetBasicInfo.lastSeen", value: "false" }
]
},
{ conjunction: AND
conditions: [
{ operator: EQUAL, path: "installationId", value: "e3" }
]
}
]
}`;
await checkFilter(null, expectedQuery);
await checkFilter(undefined, expectedQuery);
});

it('few assets', async () => {
const expectedQuery = `
{ conjunction: AND
groups: [
{ conjunction: OR
conditions: [ { operator: REGEXP, path: "assetBasicInfo.type", value: "Windows|Windows CE" } ]
},
{ conjunction: OR
conditions: [
{ operator: GREATER_THAN, path: "assetBasicInfo.lastSeen", value: "2024-10-01T22:00:00.000Z" },
{ operator: EXISTS, path: "assetBasicInfo.lastSeen", value: "false" }
]
},
{ conjunction: AND
conditions: [ { operator: EQUAL, path: "installationId", value: "e3" } ]
}
]
}`;
await checkFilter(['Windows', 'Windows CE'], expectedQuery);
});

it('many assets', async () => {
const assetTypes = [
"Automotive",
"Mobile",
"Monitor",
"NAS",
"Switch",
"Tablet",
"Toy",
"VMware Guest",
"Virtual Machine",
"Weather",
"Webserver",
"Wifi",
"Windows",
];
expect(assetTypes.join('|').length).toBeGreaterThan(LansweeperClient.maxFilterFieldLength);

const expectedFilter = `
{ conjunction: AND
groups: [
{ conjunction: OR
conditions: [
{ operator: REGEXP, path: "assetBasicInfo.type", value: "Automotive|Mobile|Monitor|NAS|Switch|Tablet|Toy|VMware Guest|Virtual Machine|Weather|Webserver|Wifi" },
{ operator: REGEXP, path: "assetBasicInfo.type", value: "Windows" }
]
},
{ conjunction: OR
conditions: [
{ operator: GREATER_THAN, path: "assetBasicInfo.lastSeen", value: "2024-10-01T22:00:00.000Z" },
{ operator: EXISTS, path: "assetBasicInfo.lastSeen", value: "false" }
]
},
{ conjunction: AND
conditions: [
{ operator: EQUAL, path: "installationId", value: "e3" }
]
}
]
}`;
await checkFilter(assetTypes, expectedFilter);
});
})
});
});
81 changes: 46 additions & 35 deletions lansweeper/aws/integration-lambda/tests/lansweeper_client_int.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,41 @@ describe.skip('integration tests', () => {
const installId = 'e3d90a6d-5f9a-4af8-8162-da011bcca979';
const siteId2 = '261986fd-78b5-4b69-ad12-a6795492f68d';

const EXPECTED_ASSET_TYPES = [
"Automotive",
"Cleaner",
"Computer",
"ESXi server",
"Energy",
"FTP server",
"Game device",
"IOS",
"Laptop",
"Linux",
"Location",
"Loudspeaker",
"Media system",
"Mobile",
"Monitor",
"NAS",
"Network device",
"Printer",
"Router",
"Server",
"Smart TV",
"Smoke",
"Surveillance Camera",
"Switch",
"Tablet",
"Toy",
"VMware Guest",
"Virtual Machine",
"Weather",
"Webserver",
"Wifi",
"Windows"
];

const testCredentials = credentials['4me'];
const accessToken = testCredentials.accessToken;
const js4meHelper = new Js4meHelper(testCredentials.env, testCredentials.account);
Expand Down Expand Up @@ -105,41 +140,7 @@ describe.skip('integration tests', () => {

it('can get asset types', async () => {
const response = await lansweeperClient.getAssetTypes(siteId1);

expect(response).toEqual([
"Alarm system",
"Automotive",
"Cleaner",
"Computer",
"ESXi server",
"Energy",
"FTP server",
"IOS",
"Laptop",
"Linux",
"Location",
"Media system",
"Mobile",
"Monitor",
"NAS",
"Network device",
"Printer",
"Router",
"Server",
"Smart Home",
"Smart TV",
"Smoke",
"Surveillance Camera",
"Switch",
"Tablet",
"Toy",
"VMware Guest",
"Virtual Machine",
"Weather",
"Webserver",
"Wifi",
"Windows"
]);
expect(response).toEqual(EXPECTED_ASSET_TYPES);
});

it('can get installation names for all sites', async () => {
Expand Down Expand Up @@ -222,6 +223,16 @@ describe.skip('integration tests', () => {
expect(filteredResults).toEqual(results);
});

it('can get assets paged for single installation and a large list of asset types', async () => {
const cutOff = new LansweeperIntegration().assetSeenCutOffDate();
const results = await lansweeperClient.getAssetsPaged(siteId1, cutOff, items => {
return items;
}, true, installId, EXPECTED_ASSET_TYPES);

console.log('assets:\n%j', results);
expect(results.length).toBeGreaterThan(0);
});

describe('performance tests', () => {
it('generate upload all different', async () => {
function createInput(run) {
Expand Down

0 comments on commit ad001f8

Please sign in to comment.