From 66886158b65ce099689f69d50aeebbd42e49db85 Mon Sep 17 00:00:00 2001 From: Daniel King Date: Fri, 12 Apr 2024 05:30:30 +0000 Subject: [PATCH] more --- llmfoundry/models/utils/param_init_fns.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llmfoundry/models/utils/param_init_fns.py b/llmfoundry/models/utils/param_init_fns.py index 62ca74c29f..d323155388 100644 --- a/llmfoundry/models/utils/param_init_fns.py +++ b/llmfoundry/models/utils/param_init_fns.py @@ -328,6 +328,8 @@ def moe_init( print(type(module)) print(isinstance(module, GLU)) print(isinstance(module, MLP)) + print(GLU) + print(MLP) if megablocks is not None and isinstance(module, ( megablocks.layers.moe.MoE, megablocks.layers.dmoe.dMoE,