From 42efbf86b694c44c6e38ca5aa7f5e39ea76e6ff1 Mon Sep 17 00:00:00 2001
From: Setra Solofoniaina
 <60129070+Setra-Solofoniaina@users.noreply.github.com>
Date: Fri, 2 Apr 2021 10:15:54 +0300
Subject: [PATCH] changed activation function to gelu

---
 src/model/bert.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/model/bert.py b/src/model/bert.py
index 22dc5a8..2334583 100644
--- a/src/model/bert.py
+++ b/src/model/bert.py
@@ -35,7 +35,7 @@ class BERT(nn.Module):
         # multi-layers transformer blocks, deep network
         #self.transformer_blocks = nn.ModuleList(
         #    [TransformerBlock(hidden, attn_heads, hidden * 4, dropout) for _ in range(n_layers)])
-        encoder_layers = nn.TransformerEncoderLayer(hidden, attn_heads, self.feed_forward_hidden, dropout)
+        encoder_layers = nn.TransformerEncoderLayer(hidden, attn_heads, self.feed_forward_hidden, dropout, activation="gelu")
         self.transformer_encoder = nn.TransformerEncoder(encoder_layers, n_layers)
 
     def _generate_square_subsequent_mask(self, sz):