Mercurial > repos > bgruening > create_tool_recommendation_model

comparison transformer_network.py @ 6:e94dc7945639 draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/bgruening/galaxytools/tree/recommendation_training/tools/tool_recommendation_model commit 24bab7a797f53fe4bcc668b18ee0326625486164
author bgruening
date Sun, 16 Oct 2022 11:52:10 +0000
parents
children
comparison
equal deleted inserted replaced
5:4f7e6612906b 6:e94dc7945639
1 import tensorflow as tf
2 from tensorflow.keras.layers import (Dense, Dropout, Embedding, Layer,
3 LayerNormalization, MultiHeadAttention)
4 from tensorflow.keras.models import Sequential
5
6
7 class TransformerBlock(Layer):
8 def __init__(self, embed_dim, num_heads, ff_dim, rate=0.1):
9 super(TransformerBlock, self).__init__()
10 self.att = MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim, dropout=rate)
11 self.ffn = Sequential(
12 [Dense(ff_dim, activation="relu"), Dense(embed_dim)]
13 )
14 self.layernorm1 = LayerNormalization(epsilon=1e-6)
15 self.layernorm2 = LayerNormalization(epsilon=1e-6)
16 self.dropout1 = Dropout(rate)
17 self.dropout2 = Dropout(rate)
18
19 def call(self, inputs, training):
20 attn_output, attention_scores = self.att(inputs, inputs, inputs, return_attention_scores=True, training=training)
21 attn_output = self.dropout1(attn_output, training=training)
22 out1 = self.layernorm1(inputs + attn_output)
23 ffn_output = self.ffn(out1)
24 ffn_output = self.dropout2(ffn_output, training=training)
25 return self.layernorm2(out1 + ffn_output), attention_scores
26
27
28 class TokenAndPositionEmbedding(Layer):
29 def __init__(self, maxlen, vocab_size, embed_dim):
30 super(TokenAndPositionEmbedding, self).__init__()
31 self.token_emb = Embedding(input_dim=vocab_size, output_dim=embed_dim, mask_zero=True)
32 self.pos_emb = Embedding(input_dim=maxlen, output_dim=embed_dim, mask_zero=True)
33
34 def call(self, x):
35 maxlen = tf.shape(x)[-1]
36 positions = tf.range(start=0, limit=maxlen, delta=1)
37 positions = self.pos_emb(positions)
38 x = self.token_emb(x)
39 return x + positions