From 4c92549538739d9b73292fe23e82a1c3c73c1d1f Mon Sep 17 00:00:00 2001 From: leihuichen <15720622991@163.com> Date: Thu, 16 Jun 2022 23:14:24 +0800 Subject: [PATCH 1/2] update core - mind (#76) --- deepmatch/layers/core.py | 74 +++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/deepmatch/layers/core.py b/deepmatch/layers/core.py index 2bdd7c2..9d020fd 100644 --- a/deepmatch/layers/core.py +++ b/deepmatch/layers/core.py @@ -159,7 +159,6 @@ def get_config(self, ): base_config = super(Similarity, self).get_config() return dict(list(base_config.items()) + list(config.items())) - class CapsuleLayer(Layer): def __init__(self, input_units, out_units, max_len, k_max, iteration_times=3, init_std=1.0, **kwargs): @@ -172,44 +171,65 @@ def __init__(self, input_units, out_units, max_len, k_max, iteration_times=3, super(CapsuleLayer, self).__init__(**kwargs) def build(self, input_shape): - self.routing_logits = self.add_weight(shape=[1, self.k_max, self.max_len], - initializer=RandomNormal(stddev=self.init_std), + self.routing_logits = self.add_weight(shape=[self.max_len, self.k_max, 1], + initializer=TruncatedNormal(stddev=self.init_std), trainable=False, name="B", dtype=tf.float32) + # N,T,k_max,1 self.bilinear_mapping_matrix = self.add_weight(shape=[self.input_units, self.out_units], - initializer=RandomNormal(stddev=self.init_std), name="S", dtype=tf.float32) super(CapsuleLayer, self).build(input_shape) def call(self, inputs, **kwargs): - behavior_embddings, seq_len = inputs + + behavior_embddings = inputs[0] + seq_len = inputs[1] batch_size = tf.shape(behavior_embddings)[0] - seq_len_tile = tf.tile(seq_len, [1, self.k_max]) + mask = tf.reshape(tf.sequence_mask(seq_len, self.max_len, tf.float32), [-1, self.max_len, 1, 1]) + + behavior_embdding_mapping = tf.matmul(behavior_embddings, self.bilinear_mapping_matrix) + behavior_embdding_mapping = tf.expand_dims(behavior_embdding_mapping, axis=2) + + behavior_embdding_mapping_ = tf.stop_gradient(behavior_embdding_mapping) # N,max_len,1,E + print(behavior_embdding_mapping_) + try: + routing_logits = tf.truncated_normal([batch_size, self.max_len, self.k_max, 1], stddev=self.init_std) + except AttributeError: + routing_logits = tf.compat.v1.truncated_normal([batch_size, self.max_len, self.k_max, 1], + stddev=self.init_std) + + k_user = None + if len(inputs) == 3: + k_user = inputs[2] + interest_mask = tf.sequence_mask(k_user, self.k_max, tf.float32) + interest_mask = tf.reshape(interest_mask, [batch_size, 1, self.k_max, 1]) + interest_mask = tf.tile(interest_mask, [1, self.max_len, 1, 1]) + + interest_padding = tf.ones_like(interest_mask) * -2 ** 31 + interest_mask = tf.cast(interest_mask, tf.bool) + + routing_logits = tf.stop_gradient(routing_logits) + self.routing_logits = routing_logits # N,max_len,k_max,1 + print(self.routing_logits) for i in range(self.iteration_times): - mask = tf.sequence_mask(seq_len_tile, self.max_len) - pad = tf.ones_like(mask, dtype=tf.float32) * (-2 ** 32 + 1) - routing_logits_with_padding = tf.where(mask, tf.tile(self.routing_logits, [batch_size, 1, 1]), pad) - weight = tf.nn.softmax(routing_logits_with_padding) - behavior_embdding_mapping = tf.tensordot(behavior_embddings, self.bilinear_mapping_matrix, axes=1) - Z = tf.matmul(weight, behavior_embdding_mapping) - interest_capsules = squash(Z) - delta_routing_logits = reduce_sum( - tf.matmul(interest_capsules, tf.transpose(behavior_embdding_mapping, perm=[0, 2, 1])), - axis=0, keep_dims=True - ) - self.routing_logits.assign_add(delta_routing_logits) + if k_user is not None: + self.routing_logits = tf.where(interest_mask, self.routing_logits, interest_padding) + weight = tf.nn.softmax(self.routing_logits, 2) * mask # N,max_len,k_max,1 + if i < self.iteration_times - 1: + Z = reduce_sum(tf.matmul(weight, behavior_embdding_mapping_), axis=1, keep_dims=True) # N,1,k_max,E + interest_capsules = squash(Z) + delta_routing_logits = reduce_sum( + interest_capsules * behavior_embdding_mapping_, + axis=-1, keep_dims=True + ) + self.routing_logits += delta_routing_logits + else: + Z = reduce_sum(tf.matmul(weight, behavior_embdding_mapping), axis=1, keep_dims=True) + interest_capsules = squash(Z) + interest_capsules = tf.reshape(interest_capsules, [-1, self.k_max, self.out_units]) return interest_capsules - def compute_output_shape(self, input_shape): - return (None, self.k_max, self.out_units) - - def get_config(self, ): - config = {'input_units': self.input_units, 'out_units': self.out_units, 'max_len': self.max_len, - 'k_max': self.k_max, 'iteration_times': self.iteration_times, "init_std": self.init_std} - base_config = super(CapsuleLayer, self).get_config() - return dict(list(base_config.items()) + list(config.items())) - def squash(inputs): vec_squared_norm = reduce_sum(tf.square(inputs), axis=-1, keep_dims=True) From fdd65e3654c6aaea64c5d933783a8a298ee3e80b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B5=85=E6=A2=A6?= Date: Fri, 17 Jun 2022 15:46:03 +0800 Subject: [PATCH 2/2] fix bug &update doc (#77) - Add Scaling factor gamma in DSSM,FM - Fix routing_logits error in MIND #44 #57 #75 --- .github/ISSUE_TEMPLATE/bug_report.md | 6 +- .github/ISSUE_TEMPLATE/question.md | 6 +- .github/workflows/ci.yml | 31 +- README.md | 11 + deepmatch/__init__.py | 2 +- deepmatch/layers/__init__.py | 6 +- deepmatch/layers/core.py | 87 ++++-- deepmatch/models/dssm.py | 11 +- deepmatch/models/fm.py | 5 +- deepmatch/models/mind.py | 47 ++- deepmatch/models/sdm.py | 7 +- deepmatch/models/youtubednn.py | 5 +- docs/pics/code2.jpg | Bin 0 -> 53499 bytes docs/pics/planet_github.png | Bin 0 -> 8309 bytes docs/requirements.readthedocs.txt | 2 + docs/source/Examples.md | 29 +- docs/source/History.md | 1 + docs/source/Quick-Start.md | 2 +- docs/source/conf.py | 2 +- docs/source/deepmatch.layers.interaction.rst | 7 + docs/source/deepmatch.layers.sequence.rst | 7 + docs/source/index.rst | 4 +- examples/colab_MovieLen1M_YoutubeDNN.ipynb | 312 +++++++------------ examples/run_sdm.py | 8 +- examples/run_youtubednn.py | 21 +- setup.py | 7 +- tests/models/MIND_test.py | 19 +- tests/models/SDM_test.py | 5 +- tests/models/YoutubeDNN_test.py | 3 +- 29 files changed, 360 insertions(+), 293 deletions(-) create mode 100644 docs/pics/code2.jpg create mode 100644 docs/pics/planet_github.png create mode 100644 docs/requirements.readthedocs.txt create mode 100644 docs/source/deepmatch.layers.interaction.rst create mode 100644 docs/source/deepmatch.layers.sequence.rst diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index f660b34..7a0a0a6 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -18,9 +18,9 @@ Steps to reproduce the behavior: 4. See error **Operating environment(运行环境):** - - python version [e.g. 3.6, 3.7] - - tensorflow version [e.g. 1.4.0, 1.14.0, 2.3.0] - - deepmatch version [e.g. 0.2.0,] + - python version [e.g. 3.6, 3.7, 3.8] + - tensorflow version [e.g. 1.4.0, 1.14.0, 2.5.0] + - deepmatch version [e.g. 0.2.1,] **Additional context** Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md index f6d65c0..2cabbcb 100644 --- a/.github/ISSUE_TEMPLATE/question.md +++ b/.github/ISSUE_TEMPLATE/question.md @@ -15,6 +15,6 @@ A clear and concise description of what the question is. Add any other context about the problem here. **Operating environment(运行环境):** - - python version [e.g. 3.6] - - tensorflow version [e.g. 1.4.0,] - - deepmatch version [e.g. 0.2.0,] + - python version [e.g. 3.6, 3.7, 3.8] + - tensorflow version [e.g. 1.4.0, 1.14.0, 2.5.0] + - deepmatch version [e.g. 0.2.1,] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 275b40d..ac474c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,25 +17,46 @@ jobs: timeout-minutes: 120 strategy: matrix: - python-version: [3.5,3.6,3.7] - tf-version: [1.4.0,1.14.0,2.1.0,2.2.0,2.3.0] + python-version: [3.6,3.7,3.8] + tf-version: [1.4.0,1.14.0,2.5.0] exclude: - python-version: 3.7 tf-version: 1.4.0 + - python-version: 3.7 + tf-version: 1.15.0 + - python-version: 3.8 + tf-version: 1.4.0 + - python-version: 3.8 + tf-version: 1.14.0 + - python-version: 3.8 + tf-version: 1.15.0 + - python-version: 3.6 + tf-version: 2.7.0 + - python-version: 3.6 + tf-version: 2.8.0 + - python-version: 3.6 + tf-version: 2.9.0 + - python-version: 3.9 + tf-version: 1.4.0 + - python-version: 3.9 + tf-version: 1.15.0 + - python-version: 3.9 + tf-version: 2.2.0 steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 - name: Setup python environment - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | pip3 install -q tensorflow==${{ matrix.tf-version }} + pip install -q protobuf==3.19.0 pip install -q requests pip install -e . - name: Test with pytest @@ -46,7 +67,7 @@ jobs: pip install -q python-coveralls pytest --cov=deepmatch --cov-report=xml - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1.0.2 + uses: codecov/codecov-action@v3.1.0 with: token: ${{secrets.CODECOV_TOKEN}} file: ./coverage.xml diff --git a/README.md b/README.md index 75e184e..69dea2a 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,12 @@ Let's [**Get Started!**](https://deepmatch.readthedocs.io/en/latest/Quick-Start. Wang Zhe ​

Baidu Inc.

​ + + ​ pic
+ ​ Chen Leihui ​ +

+ Alibaba Group

​ + ​ pic
LeoCai @@ -57,6 +63,11 @@ Let's [**Get Started!**](https://deepmatch.readthedocs.io/en/latest/Quick-Start. ​ Yang Jieyu

Ant Group

​ + + ​ pic
+ ​ Meng Yifan +

DeepCTR

​ + diff --git a/deepmatch/__init__.py b/deepmatch/__init__.py index 0fa3e17..6ed5791 100644 --- a/deepmatch/__init__.py +++ b/deepmatch/__init__.py @@ -1,4 +1,4 @@ from .utils import check_version -__version__ = '0.2.0' +__version__ = '0.2.1' check_version(__version__) diff --git a/deepmatch/layers/__init__.py b/deepmatch/layers/__init__.py index 854c596..a6053ed 100644 --- a/deepmatch/layers/__init__.py +++ b/deepmatch/layers/__init__.py @@ -1,7 +1,8 @@ from deepctr.layers import custom_objects from deepctr.layers.utils import reduce_sum -from .core import PoolingLayer, Similarity, LabelAwareAttention, CapsuleLayer, SampledSoftmaxLayer, EmbeddingIndex +from .core import PoolingLayer, Similarity, LabelAwareAttention, CapsuleLayer, SampledSoftmaxLayer, EmbeddingIndex, \ + MaskUserEmbedding from .interaction import DotAttention, ConcatAttention, SoftmaxWeightedSum, AttentionSequencePoolingLayer, \ SelfAttention, \ SelfMultiHeadAttention, UserAttention @@ -23,7 +24,8 @@ 'SelfAttention': SelfAttention, 'SelfMultiHeadAttention': SelfMultiHeadAttention, 'UserAttention': UserAttention, - 'DynamicMultiRNN': DynamicMultiRNN + 'DynamicMultiRNN': DynamicMultiRNN, + 'MaskUserEmbedding': MaskUserEmbedding } custom_objects = dict(custom_objects, **_custom_objects) diff --git a/deepmatch/layers/core.py b/deepmatch/layers/core.py index 9d020fd..1267653 100644 --- a/deepmatch/layers/core.py +++ b/deepmatch/layers/core.py @@ -1,7 +1,7 @@ import tensorflow as tf from deepctr.layers.activation import activation_layer from deepctr.layers.utils import reduce_max, reduce_mean, reduce_sum, concat_func, div, softmax -from tensorflow.python.keras.initializers import RandomNormal, Zeros, glorot_normal +from tensorflow.python.keras.initializers import RandomNormal, Zeros, TruncatedNormal from tensorflow.python.keras.layers import Layer from tensorflow.python.keras.regularizers import l2 @@ -103,19 +103,19 @@ def call(self, inputs, training=None, **kwargs): weight = tf.pow(weight, self.pow_p) # [x,k_max,1] if len(inputs) == 3: - k_user = tf.cast(tf.maximum( - 1., - tf.minimum( - tf.cast(self.k_max, dtype="float32"), # k_max - tf.log1p(tf.cast(inputs[2], dtype="float32")) / tf.log(2.) # hist_len - ) - ), dtype="int64") + k_user = inputs[2] seq_mask = tf.transpose(tf.sequence_mask(k_user, self.k_max), [0, 2, 1]) padding = tf.ones_like(seq_mask, dtype=tf.float32) * (-2 ** 32 + 1) # [x,k_max,1] weight = tf.where(seq_mask, weight, padding) - weight = softmax(weight, dim=1, name="weight") - output = reduce_sum(keys * weight, axis=1) + if self.pow_p >= 100: + idx = tf.stack( + [tf.range(tf.shape(keys)[0]), tf.squeeze(tf.argmax(weight, axis=1, output_type=tf.int32), axis=1)], + axis=1) + output = tf.gather_nd(keys, idx) + else: + weight = softmax(weight, dim=1, name="weight") + output = tf.reduce_sum(keys * weight, axis=1) return output @@ -159,6 +159,7 @@ def get_config(self, ): base_config = super(Similarity, self).get_config() return dict(list(base_config.items()) + list(config.items())) + class CapsuleLayer(Layer): def __init__(self, input_units, out_units, max_len, k_max, iteration_times=3, init_std=1.0, **kwargs): @@ -171,32 +172,28 @@ def __init__(self, input_units, out_units, max_len, k_max, iteration_times=3, super(CapsuleLayer, self).__init__(**kwargs) def build(self, input_shape): - self.routing_logits = self.add_weight(shape=[self.max_len, self.k_max, 1], - initializer=TruncatedNormal(stddev=self.init_std), - trainable=False, name="B", dtype=tf.float32) - # N,T,k_max,1 self.bilinear_mapping_matrix = self.add_weight(shape=[self.input_units, self.out_units], name="S", dtype=tf.float32) super(CapsuleLayer, self).build(input_shape) def call(self, inputs, **kwargs): - behavior_embddings = inputs[0] + behavior_embedding = inputs[0] seq_len = inputs[1] - batch_size = tf.shape(behavior_embddings)[0] + batch_size = tf.shape(behavior_embedding)[0] mask = tf.reshape(tf.sequence_mask(seq_len, self.max_len, tf.float32), [-1, self.max_len, 1, 1]) - behavior_embdding_mapping = tf.matmul(behavior_embddings, self.bilinear_mapping_matrix) - behavior_embdding_mapping = tf.expand_dims(behavior_embdding_mapping, axis=2) + behavior_embedding_mapping = tf.tensordot(behavior_embedding, self.bilinear_mapping_matrix, axes=1) + behavior_embedding_mapping = tf.expand_dims(behavior_embedding_mapping, axis=2) - behavior_embdding_mapping_ = tf.stop_gradient(behavior_embdding_mapping) # N,max_len,1,E - print(behavior_embdding_mapping_) + behavior_embdding_mapping_ = tf.stop_gradient(behavior_embedding_mapping) # N,max_len,1,E try: routing_logits = tf.truncated_normal([batch_size, self.max_len, self.k_max, 1], stddev=self.init_std) except AttributeError: routing_logits = tf.compat.v1.truncated_normal([batch_size, self.max_len, self.k_max, 1], stddev=self.init_std) + routing_logits = tf.stop_gradient(routing_logits) k_user = None if len(inputs) == 3: @@ -208,13 +205,14 @@ def call(self, inputs, **kwargs): interest_padding = tf.ones_like(interest_mask) * -2 ** 31 interest_mask = tf.cast(interest_mask, tf.bool) - routing_logits = tf.stop_gradient(routing_logits) - self.routing_logits = routing_logits # N,max_len,k_max,1 - print(self.routing_logits) for i in range(self.iteration_times): if k_user is not None: - self.routing_logits = tf.where(interest_mask, self.routing_logits, interest_padding) - weight = tf.nn.softmax(self.routing_logits, 2) * mask # N,max_len,k_max,1 + routing_logits = tf.where(interest_mask, routing_logits, interest_padding) + try: + weight = softmax(routing_logits, 2) * mask + except TypeError: + weight = tf.transpose(softmax(tf.transpose(routing_logits, [0, 1, 3, 2])), + [0, 1, 3, 2]) * mask # N,max_len,k_max,1 if i < self.iteration_times - 1: Z = reduce_sum(tf.matmul(weight, behavior_embdding_mapping_), axis=1, keep_dims=True) # N,1,k_max,E interest_capsules = squash(Z) @@ -222,18 +220,27 @@ def call(self, inputs, **kwargs): interest_capsules * behavior_embdding_mapping_, axis=-1, keep_dims=True ) - self.routing_logits += delta_routing_logits + routing_logits += delta_routing_logits else: - Z = reduce_sum(tf.matmul(weight, behavior_embdding_mapping), axis=1, keep_dims=True) + Z = reduce_sum(tf.matmul(weight, behavior_embedding_mapping), axis=1, keep_dims=True) interest_capsules = squash(Z) interest_capsules = tf.reshape(interest_capsules, [-1, self.k_max, self.out_units]) return interest_capsules + def compute_output_shape(self, input_shape): + return (None, self.k_max, self.out_units) + + def get_config(self, ): + config = {'input_units': self.input_units, 'out_units': self.out_units, 'max_len': self.max_len, + 'k_max': self.k_max, 'iteration_times': self.iteration_times, "init_std": self.init_std} + base_config = super(CapsuleLayer, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + def squash(inputs): vec_squared_norm = reduce_sum(tf.square(inputs), axis=-1, keep_dims=True) - scalar_factor = vec_squared_norm / (1 + vec_squared_norm) / tf.sqrt(vec_squared_norm + 1e-8) + scalar_factor = vec_squared_norm / (1 + vec_squared_norm) / tf.sqrt(vec_squared_norm + 1e-9) vec_squashed = scalar_factor * inputs return vec_squashed @@ -255,3 +262,27 @@ def get_config(self, ): config = {'index': self.index, } base_config = super(EmbeddingIndex, self).get_config() return dict(list(base_config.items()) + list(config.items())) + + +class MaskUserEmbedding(Layer): + + def __init__(self, k_max, **kwargs): + self.k_max = k_max + super(MaskUserEmbedding, self).__init__(**kwargs) + + def build(self, input_shape): + super(MaskUserEmbedding, self).build( + input_shape) # Be sure to call this somewhere! + + def call(self, x, training=None, **kwargs): + user_embedding, interest_num = x + if not training: + interest_mask = tf.sequence_mask(interest_num, self.k_max, tf.float32) + interest_mask = tf.reshape(interest_mask, [-1, self.k_max, 1]) + user_embedding *= interest_mask + return user_embedding + + def get_config(self, ): + config = {'k_max': self.k_max, } + base_config = super(MaskUserEmbedding, self).get_config() + return dict(list(base_config.items()) + list(config.items())) diff --git a/deepmatch/models/dssm.py b/deepmatch/models/dssm.py index 1ac95cd..f00eb3f 100644 --- a/deepmatch/models/dssm.py +++ b/deepmatch/models/dssm.py @@ -15,8 +15,8 @@ def DSSM(user_feature_columns, item_feature_columns, user_dnn_hidden_units=(64, 32), item_dnn_hidden_units=(64, 32), - dnn_activation='tanh', dnn_use_bn=False, - l2_reg_dnn=0, l2_reg_embedding=1e-6, dnn_dropout=0, seed=1024, metric='cos'): + dnn_activation='relu', dnn_use_bn=False, + l2_reg_dnn=0, l2_reg_embedding=1e-6, dnn_dropout=0, gamma=10, seed=1024, metric='cos'): """Instantiates the Deep Structured Semantic Model architecture. :param user_feature_columns: An iterable containing user's features used by the model. @@ -28,6 +28,7 @@ def DSSM(user_feature_columns, item_feature_columns, user_dnn_hidden_units=(64, :param l2_reg_dnn: float. L2 regularizer strength applied to DNN :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate. + :param gamma: float. Scaling factor. :param seed: integer ,to use as random seed. :param metric: str, ``"cos"`` for cosine or ``"ip"`` for inner product :return: A Keras model instance. @@ -55,12 +56,12 @@ def DSSM(user_feature_columns, item_feature_columns, user_dnn_hidden_units=(64, item_dnn_input = combined_dnn_input(item_sparse_embedding_list, item_dense_value_list) user_dnn_out = DNN(user_dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, - dnn_use_bn, seed=seed)(user_dnn_input) + dnn_use_bn, output_activation='linear', seed=seed)(user_dnn_input) item_dnn_out = DNN(item_dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, - dnn_use_bn, seed=seed)(item_dnn_input) + dnn_use_bn, output_activation='linear', seed=seed)(item_dnn_input) - score = Similarity(type=metric, gamma = 10)([user_dnn_out, item_dnn_out]) + score = Similarity(type=metric, gamma=gamma)([user_dnn_out, item_dnn_out]) output = PredictionLayer("binary", False)(score) diff --git a/deepmatch/models/fm.py b/deepmatch/models/fm.py index 97bf339..5b25fe0 100644 --- a/deepmatch/models/fm.py +++ b/deepmatch/models/fm.py @@ -8,12 +8,13 @@ from ..layers.core import Similarity -def FM(user_feature_columns, item_feature_columns, l2_reg_embedding=1e-6, seed=1024, metric='cos'): +def FM(user_feature_columns, item_feature_columns, l2_reg_embedding=1e-6, gamma=10, seed=1024, metric='cos'): """Instantiates the FM architecture. :param user_feature_columns: An iterable containing user's features used by the model. :param item_feature_columns: An iterable containing item's features used by the model. :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector + :param gamma: float. Scaling factor. :param seed: integer ,to use as random seed. :param metric: str, ``"cos"`` for cosine or ``"ip"`` for inner product :return: A Keras model instance. @@ -46,7 +47,7 @@ def FM(user_feature_columns, item_feature_columns, l2_reg_embedding=1e-6, seed=1 item_dnn_input = concat_func(item_sparse_embedding_list, axis=1) item_vector_sum = Lambda(lambda x: reduce_sum(x, axis=1, keep_dims=False))(item_dnn_input) - score = Similarity(type=metric)([user_vector_sum, item_vector_sum]) + score = Similarity(type=metric, gamma=gamma)([user_vector_sum, item_vector_sum]) output = PredictionLayer("binary", False)(score) diff --git a/deepmatch/models/mind.py b/deepmatch/models/mind.py index 3053dd1..556e318 100755 --- a/deepmatch/models/mind.py +++ b/deepmatch/models/mind.py @@ -11,12 +11,13 @@ embedding_lookup, varlen_embedding_lookup, get_varlen_pooling_list, get_dense_input, build_input_features from deepctr.layers import DNN from deepctr.layers.utils import NoMask, combined_dnn_input -from tensorflow.python.keras.layers import Concatenate +from tensorflow.python.keras.layers import Concatenate, Lambda from tensorflow.python.keras.models import Model -from deepmatch.utils import get_item_embedding from ..inputs import create_embedding_matrix -from ..layers.core import CapsuleLayer, PoolingLayer, LabelAwareAttention, SampledSoftmaxLayer, EmbeddingIndex +from ..layers.core import CapsuleLayer, PoolingLayer, MaskUserEmbedding, LabelAwareAttention, SampledSoftmaxLayer, \ + EmbeddingIndex +from ..utils import get_item_embedding def shape_target(target_emb_tmp, target_emb_size): @@ -27,7 +28,24 @@ def tile_user_otherfeat(user_other_feature, k_max): return tf.tile(tf.expand_dims(user_other_feature, -2), [1, k_max, 1]) -def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1.0, dynamic_k=False, +def adaptive_interest_num(seq_len, k_max): + try: + log_len = tf.log1p(tf.cast(seq_len, dtype="float32")) + log_2 = tf.log(2.) + except AttributeError: + log_len = tf.math.log1p(tf.cast(seq_len, dtype="float32")) + log_2 = tf.math.log(2.) + k_user = tf.cast(tf.maximum( + 1., + tf.minimum( + tf.cast(k_max, dtype="float32"), # k_max + log_len / log_2 # hist_len + ) + ), dtype="int32") + return k_user + + +def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=100, dynamic_k=True, user_dnn_hidden_units=(64, 32), dnn_activation='relu', dnn_use_bn=False, l2_reg_dnn=0, l2_reg_embedding=1e-6, dnn_dropout=0, output_activation='linear', seed=1024): """Instantiates the MIND Model architecture. @@ -110,14 +128,20 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1 # max_len = history_emb.get_shape()[1].value hist_len = features['hist_len'] - high_capsule = CapsuleLayer(input_units=item_embedding_dim, - out_units=item_embedding_dim, max_len=seq_max_len, - k_max=k_max)((history_emb, hist_len)) + if dynamic_k: + interest_num = Lambda(adaptive_interest_num, arguments={'k_max': k_max})(hist_len) + high_capsule = CapsuleLayer(input_units=item_embedding_dim, + out_units=item_embedding_dim, max_len=seq_max_len, + k_max=k_max)((history_emb, hist_len, interest_num)) + else: + high_capsule = CapsuleLayer(input_units=item_embedding_dim, + out_units=item_embedding_dim, max_len=seq_max_len, + k_max=k_max)((history_emb, hist_len)) if len(dnn_input_emb_list) > 0 or len(dense_value_list) > 0: user_other_feature = combined_dnn_input(dnn_input_emb_list, dense_value_list) - other_feature_tile = tf.keras.layers.Lambda(tile_user_otherfeat, arguments={'k_max': k_max})(user_other_feature) + other_feature_tile = Lambda(tile_user_otherfeat, arguments={'k_max': k_max})(user_other_feature) user_deep_input = Concatenate()([NoMask()(other_feature_tile), high_capsule]) else: @@ -125,7 +149,7 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1 user_embeddings = DNN(user_dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, dnn_use_bn, output_activation=output_activation, seed=seed, - name="user_embedding")( + name="user_dnn")( user_deep_input) item_inputs_list = list(item_features.values()) @@ -138,9 +162,10 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1 pooling_item_embedding_weight = PoolingLayer()([item_embedding_weight]) if dynamic_k: - user_embedding_final = LabelAwareAttention(k_max=k_max, pow_p=p, )((user_embeddings, target_emb, hist_len)) + user_embeddings = MaskUserEmbedding(k_max)([user_embeddings, interest_num]) + user_embedding_final = LabelAwareAttention(k_max=k_max, pow_p=p)((user_embeddings, target_emb, interest_num)) else: - user_embedding_final = LabelAwareAttention(k_max=k_max, pow_p=p, )((user_embeddings, target_emb)) + user_embedding_final = LabelAwareAttention(k_max=k_max, pow_p=p)((user_embeddings, target_emb)) output = SampledSoftmaxLayer(num_sampled=num_sampled)( [pooling_item_embedding_weight, user_embedding_final, item_features[item_feature_name]]) diff --git a/deepmatch/models/sdm.py b/deepmatch/models/sdm.py index 6d6ca96..5fbd5c4 100644 --- a/deepmatch/models/sdm.py +++ b/deepmatch/models/sdm.py @@ -15,10 +15,10 @@ from tensorflow.python.keras.layers import Dense, Lambda from tensorflow.python.keras.models import Model -from deepmatch.utils import get_item_embedding from ..layers.core import PoolingLayer, SampledSoftmaxLayer, EmbeddingIndex from ..layers.interaction import UserAttention, SelfMultiHeadAttention, AttentionSequencePoolingLayer from ..layers.sequence import DynamicMultiRNN +from ..utils import get_item_embedding def SDM(user_feature_columns, item_feature_columns, history_feature_list, num_sampled=5, units=64, rnn_layers=2, @@ -156,8 +156,3 @@ def SDM(user_feature_columns, item_feature_columns, history_feature_list, num_sa get_item_embedding(pooling_item_embedding_weight, item_features[item_feature_name])) return model - # , Model(inputs=user_inputs_list, outputs=gate_output_reshape), Model(inputs=item_inputs_list, - # outputs=get_item_embedding( - # pooling_item_embedding_weight, - # item_features[ - # item_feature_name])) diff --git a/deepmatch/models/youtubednn.py b/deepmatch/models/youtubednn.py index 3b96a70..f146e87 100644 --- a/deepmatch/models/youtubednn.py +++ b/deepmatch/models/youtubednn.py @@ -9,10 +9,9 @@ from deepctr.layers.utils import NoMask, combined_dnn_input from tensorflow.python.keras.models import Model -from deepmatch.layers import PoolingLayer -from deepmatch.utils import get_item_embedding from ..inputs import input_from_feature_columns, create_embedding_matrix -from ..layers.core import SampledSoftmaxLayer, EmbeddingIndex +from ..layers.core import SampledSoftmaxLayer, EmbeddingIndex, PoolingLayer +from ..utils import get_item_embedding def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, diff --git a/docs/pics/code2.jpg b/docs/pics/code2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e191f2971ebdfcc60981e4b680e3331735874744 GIT binary patch literal 53499 zcmd43c_5VE`!{}%8T-EPMzWO<*|Lq2ElE)cA=yL9t}s&8EJY~FRFWj-oi&oNOA;bN z_OdTCVN7Owj^69%^ZEXM-|zGLJ%2p6?zvxc?sK1WUDrA1I?;7(5FkSb=m-cn1Ur1bGKsmOh}eAJRE& zVTIT!5gfm;+%L#8Ael0p4wWMoVp#7jlMYAIr+HW`jE!AR*jSrd95eZ)6gSEB@?}2; z9tiTk6dY({c3AqXy@NFKERb9R;)ZrXdtBXuE+0B^;@D1^e_nr;|JMg`a7S|p>Q~+= z>#yYh8Q^pG2yz2fg=mHP*3^d)*eJ6ji ze-J>wbca9k_OaM`7T5t^9_)R}9AID{c&nT1FZqEk@LrEghyJh$Jk%rT?2c?8KSQ;< z#}N~Nf$s2mk4vX_^xY|kbUFC&PF>(x1lwi5U*iO%BV;`M%y!ac0M-Z!wf*B+#F^m0 zQ#;1fGG1acy-UUWTr0$?By;x=>)GKP*o(ohIEFF*m%Mabvla?LXM_WK=E z$Q25NydV!q`Hz%8pRoIN=Mwm|g|0#?&?S&3=&$Dv|GMh|g@SbIU#Y)8s|Ifag!9Vr*J3r$rV;|!*<7dW6 z#$gwq*ua0YkrgxvH2U3!9>Bi;umwUMl&Oh0j4%MDgOU+K2r)<*p#kbSgg}9sH9@NJ z9~$mh<f8=A`&&t3m%xc7{`0r10?d8(^Q{P{<{--5>m-WJ5_Wo0X z|E&M-J3f#*kkj(7*7!pLcp|(4-T`ldcf#x8HIOtM2Y(I!0DrrK|ES;eSL=HHRh!K( zd3-^;`23aUPksMv@tyt=w7+Ek!hXu{r38C~2ZR0$83$aB2=wvt4wgOyW(N;xi%V`Q z`=r(PtE)rM&RnyT2ZGjxe$62;q4$5M-2h|U;1Zoqum3yEavOpwcY)zQ`FEPaC*UT4 zpMH18EhI4X_wo=s7e?SExWPyhg`}X}kRs4W9nymIAS1{GItp1sryzUC8T40g(9Z** zFennb4#h*apxaOulnY^?0_ZtZ3Y9}OP(9QHwL@J{FZ3B21-<7xv;^THB1DG4V9YR1 z7(YxDhJ?w%lwj&G9hf1^1a=H&1G9&nhk3y+!9rk>up6*junbr(>@lnu_6k-DYl3yc z`e4JbDcAz+2W*=G&cMMSz#z#W&!Ebn!*GbflHnAC6N4AS6^5$}HyP3x?lC-Jc*#)B z(8SQi@R?zfVToZA4#C;sLhxO1Ww;L97;X)BfP2CN;nDD9csBeAybN9k+PNP-315c) zL@*)v5z+`{(1Xnprx9+5D~M=B3L+Qr98ryEMf4-45O@TQk&{t^QHfEP(Sq?TqZeZs zV?1Lv<1@x8#x^jPzB3Y;n3#l^6qvNZShQ!l$aIYp7}oWOJ*GNC+6?WBo=lSB#SzWDT@QkC6-v0Y?c=+4J@Bn=2*zAJgjo8 zx~$f$?yM24>8wv#>sWhOzq69rc-a)#4A@SwU1Yn?c8{%$t&Qy~+d4ZtyA1n5_LJ;B z>@n>3*neYx$3DgWlY@_AFNZP5IgSvHG>#&UMvh^QbxuxBc}^ov2hL#5G|pnq7S3@_ z5*I(03YP_!I~STOkE@#N6W1y?E4Li?A#P{xtK8Y#72G}C%RH<+@;t^ot~}8^c|5f| zLp+z(U}nz%7ANfsX=sK|VnZ!83y4f_Z{(1t*2zLh?cuLViN&Lghk(LfgWU!iR)C zgp-8Jgg*&yiHM6BiFk-4i(o}Qi)@QZi<*e~irx{e5gijli0u_SDHbM{FV-fuEG{6f zC+;SmEM6`?BEcZBSHec(nnZ!b2Z>EdDM>TQ%aZpcTO^mIgrtn5E=t{%YLHq$@*@qA zKFGUB9CA@wQ2LOxpY%QHR%!e$iCyNqLUujg^>G(jMnUGZOsou6W^6axZk^qpyYKF9 z-i?<<%38@r%9hBE$g#@l$a%}<$aTnV$t%d)$tTFy$j>W?C|D?5RVY#Ts>rE`QoN%0 zL~&pb(;l5YzIz_->D|k)S97oT-n_j%N(@R`Np0_Y%pmkZFt`Bk>Mn2 z7s?g&1T}56+sNJMnbF*#J%=tHdUvdHrr9;KMsr4UOY=MCLl%-2ZWhHB_@mlKqmH&(a#-40=2}i4+j}hV zSe+Gv)lsWVtFOo9j$b-nV+~nbSZ7*~olrP&<;0tlOeasA%sV-2qh=Fn(_zbR>tg%D zmU!y$sq|B$rxj0!oNhkDedgSm;xk)zN9;1}CeNy#jXK+9FJ|v!|Js4s;f%vGhfPNl z$1KP1=QPj7pBr>ia0++obQW{=bAIc>?c(ZEah~zK-T4>iDX!M8Ph2-ISX_8;0qXZ`VS$B zA#ov7p@yM(p`@@gVHM%L;g`ZcMC^@7ikQ1fJE%L)Gk7zu zX3S(-WmeynzMFh^Bg-YLGg~7Ylf#x1k~4Yl*uB^HW$)j1moSuslq|eF_p-P2Na^b`<+8_EA?&T+px;7%TYPo?)nK`0d2@wM zMOmd><%25zs^n@|bwoA3#=BK4t3z(8N93S)a|T&uk*g9ORKB; zgXV{-Zq4qhk6ItAd$fCA_a5x6>(lRR_=Nh@)PJPEeZXR%>+|u?eS@b5zYLuln*8Ga zWq#Oim@pDLLLQCz%J?;TjAtx+Tw?sm#GVQ4q}JrysUuS#rfsK3XWV9%z6E`we7`x% zIh#F)oGYGJpRZpqUFcnOT>QRtd5OFnzrwrnU{!Ioa?NP%1O6=jJ0Xxj|BdH?IDQ27w`wi-($gFIeta%e;^Pd{X)NA|ABvx7k}N;f}k=0c4)kDBLw{h zPWc#6HaK@E{3#=d;lP2vPLQ&51bH3+j0YSrjzB<7NObzTBm^ zfX%>RG7tkV49*Lqw?arD8zUez;P|r=7y}%^$i&RT%Ek^dRPjIzFgTn60cT{~S#W?w zf%_1GmyvIm+94+XldjA%R|M3r-+9ck`|ztKL7P6j?12k`v8-%D!Xlz#a`Fm_d-iH* zYH8~n)HOb0Vrph?an$zI=`(hK%DK6FczSvJTnq{h2@MO6xO(GeTzo=eQgTM--K^}K zd-rpn6g(|_R`k61MR`SKRdvnl+PdbJ*0%PJcb)G)^$&a=9QramGBrK(?fdN9{K6vP z$NI)5af|eGdq*!A1pi&Fzcl+dy?B9M36JFIQFgGGO1QcbD&U zh()K0LrrR<9g=Uy^9or;*eyTbYcCg7 z*{^cS{Av43<|-)a1&e{Ag5WxOxt8F(HJ#I3B1EwGq3=Hx=Fi_cm}ZpC>D;nqTOlp` z;<(+%M9f?9V(cXXDw(1^ImWftny)gjQt7m}Nz8r~D|gm4l}E_a3ye=TXg-WB>}~Yv zd5mbZf<47-0O!|jdF=CQ1JhDpSzJKTQ*G1jmZQ~r1q9yy4rPw@H4cQlD_pjgjcsVb z4A!@F!7fosyyRep_rtU9JnyW(560*QzSn(V!^*RvNl;y}oc>AC zCU8$h=M{%{IN<~0L(D$oD(Cv#yu9|0mKe!-)V%(p{}I;Msner_eGsiqF~jF4;4y8v zJuS#IXT1S^N3UmJtqL9M1J%2)-nw_Yxc@HChyA8z(I!xIRo{stbDb#8Ca{a&O zW2A-Ej&TGkhFZYh?KL_N+xB>ETSCZixz(_zlH}%3HZXfhDxWicLtuAE7cibm=X+M{ zHc3jXJ?p7j^b#9MhX~&?j+{-TLy;el-?5*$<1jq+;)DZDLgTwgQP+a56&_jtvacq4 zOKVHXASE?G-AZuLOy`cQsolB&U*l|yfg;b<6KA=q@TtTqf^Z8R;tig-LEBAzyz3Pq zw8T?od9oHiR#@kvxKGl*Fw$gSQjIF#!4Mmf6doT#b=1mkd``2L^ZtbCr&l|pf)4dN zeBMyfxzX`rF?3Kpa_QYJPuXk95V@Ym)I>btXyLUe-@-3F&t2Z_s;*4^o^3j1qtoQ+ zwhVPg`V&T{rIHBnDRgs>MC8GtOR~N77BfiBdg0JOEAJtp@~@KbCaw?-_Y-~-6-At2 z9%;PsDb{G|T>PCG9FuRf;!vb$!P1$Wy0!R*k0E*@9(MWfocCmK8^GR2B(tkrO}l!b z!I)auuxo^LV>;{xjcvhMDS*S#yG-9{y4!RF?_lNebm9KBYggn=W#3}p0pT__*WBT& z^~vhGo;WRKLmi4SX>Hm%zJe6gHj>cf!X4hyoI74$qTFyN>`LIiwGGm~!m%j9=Dk*o zUFHI=uTpLdzAYfS5in_+IC083ymd;X=lH=Y{PEHvd!Ox_*@I_(SO|X1WBeLNYmqpq zwl1f1yPgpa&Bhou{$uq2=?UQU-hcHB&t5gYD>^wCV|bB7hwv_KE|SQWah}%%Gop42 zu2Qt#@|ok|<}Lg9eHv%b7snD;o?f{X$6DLJSA@a0Wjf&)X@YFYX?^5+8BL1z@r0!> z4G0$-b~V!?!e+EewKEprCE$7>%`5LuEX61x?lpPqSw56`M@!Z`0DQjy+ptY3Zrf zgZj7h6HGdHRAW*R-h#+`!Zp&TC>OqvQ$pvdGSlP8vmb^c%b?yUw~ za=a#9lEHfRTpVwKmhhD^BPm9*HLX|3ZsHXu9csm1Z%}wWh^^u;wD;L^I2NyLYCpax zSeqosQy<14cembOxU1@hJWOx{+oYP{!Vvt%r$L1x+^0bnp-DT8I$K{WdY-Dvdp+d# z6H0Pxt;*q8$F(45pC{3s)oQd)x;Y&#JWlp1c>%iP^V?eYy$A28Jnhx2ugiWk`sLi% z&*OdVMyF28RpD#kQtUt)*@bc-6%3<#?`og*TYyPe$dVql*=6l9GZ!=T>5y|yyP-%gmSTd9M3K6gFolj?K|u#dZ;m#%v1Nzt&AqP=S+E(# z;!b_HIq?E-EhLnB8U_EhA(< zt?-%s&@&UOW*sqst|>H5uzm57;;A7RVoH6tpGqlJl@AChqiu3*vIi_#TqXQ>oqwSH zQDE{;?v_}ohmXIj7h~#!3nI#Q+){;Op2zIHz6`b5KVnE${2!l8wIhYE9&ilJDlOs| zLnQb3ZKQKWy5qYm>*9+`ETl3$ru}6*!%D1)w{r59o7HDNbf##X*4FyK;Jfjo)9Hd)y?yDjq+Py1{NXVCa_CclT8l)vrSuU9vhOtT9lc_N%p2Ds^-pn+^taeWGm_AF zJ4aGp!G~g5hcYaCc#v7V+Z(@EE>3>+_Pag4m%4hMOEbNQYwNO38!?LsE}2Mkpxp|g zL#)iH6h_}7?>fnsNxsimLgqbhi`RbX4w1d}N;KK@kxA{TOPbqOm3ya5p?RfmNVb9?Z^NGO8*RGG37eT^KG9OSdOG|3SKU#=5Ik#$MW<8$O`HSAVgG zUuUFEB&B$-xlX&8q|%(~EexxnLZmh6#?&4eg8aiG61-(Q9#hhD-p@VLHg~((Lh`T^ z>k%J*4|R@Xf!y6U8kzGcSCgW7D50bv@{zM89YsradnU2`KhNLwHLzHJ_Vrx>Tx;u% z8%I!;*QH_2DLPb*Z5g$eqMX5Z+}xOm(U#IBhgS{MADULuQ`Z05>2_i5+d}qGO}VK| zgu4mjC>m$sz0-zS3tn;83IbgW|c@`Zrv(DCQ*InSf*|t!%O_sTezN z$a>8&Jnj19d%?sE@8Mjt%o9hn54rz5boL=j)6d#R+;<}bq}~uvGcGM&gr3Gy)af^L z=q9QSPUAHQT{~Wz_DMX-_Gp@T=B~7B0S7K=J#l~0pnbXdtpZWylXP#iTx4|LO8st+ zN%NI@<3h&1;C0rj3TD$vo%H1eiL;SE6E1R{ol}rq3lAXbf2JYBU-Y<;+z4EAE!9^I z%8Fc{4Qu5EL{<}X-wtTkSjrBm^q)Jr-k9QBpV5Tit zXMQ~U@Xbh!q>22Ro9UXH5xa@Ts}6=-?45qhf)U?I^W%*B4;nmYIJ)P}+hTa;r~tIR z=jEvBm0Y8r2Aq&#va8#d6B1%jl)yo@3ixs-4k2t+*`$ zuyXo8u{(;CtVV}0$e-F2IMMfm^+~gP`A=}7(ZdmR2*pQ-COYVlnBl*CGq{azJ(*4c zyrmO2iL7wGa5Y)KbyJYkMeOSNNy?$oP^7+pz~vh>nH2Q@j!Z~`fEyh$MEwXLF)t4R zd4SbSqSdM8>$GS&I)qE7X+-@4Uz4Uoi3P}?K)FnGBg(VBbyFbX|Jh0f!^{*D5jxb9 zPRXOe$eO@F1we0?6r5i@utXAiZH5k|BLDeC>K$rYv4a+g{x*T<9>xM`q=3>C53{0Xe<1f?ud+49=>7%!%(d}fHQT8?|I4NQ}^&)>n#IsTRVF#Da0G{kw z^~a}At%%>_YN%o*!RQd9wp%yu77BkARd8yel4eE`sPv~pk270bjMs-&y;eNPCdlC% zQ%5P5V!{;JgSe`4vf@2Q%trQz$U(%FwV%llWiP6N4p9WYp}T5saT)A*{Q`dEDa1Njn8Ydr8v+xez6v^b+mAcD;YTWBOUibQBwB;F)B&O8Jk7IHtLd?Bn|n z&f=c9e3@yp%fXdvxAcgF<7_e(BQ(s*u&OE#-c$<*y{YX)*}7nm__LFYxaJhlOIPR+ zElHDzTCQZJL$OgLqfI)LY=Yacc*OmPCQAE+h8wCaZixvugK_FZlf_jf^kj=4jO;L9 zPyL|BL2yZQ!JkH^Ao(e9vUyqF4GN{&k?F_uY?VZxZadQRn z;P(ClYc7v5qK52d1mDRsSiFtX>B{e)D@m1GZ+X`|PMx-CVudo(?ytGrTX$}uLx@M8 z`mL_ZlI|MHI*A@~lgb?|EgFp*r5tU_3P-7MLQw=l4Y8VDi29p)B~ zZp}tlVd6AVBcQ$b^)MXRG>xjE__XTI%eVciPP`IL;uvO8H14>K!1AJPl)8H0>rfL0Z}FQg(ekFCUe3AC zfrc3vI&?bxcnSGD9cu9lzhOsBEz_4%h;t*H?xx+_{xl}{E`eE%2VL^!B*jhz1Alr-?%TWOh(eu@ z$MrV!(B;^Uhj3yL$=C-!L&OnUqs2+LB)_kG9jyC~&&Ch;>>+F9{b($Zp}f{Vp8{`O zMY&sl(M1m1w&xC|>?S-Im)C8(87jZW--cV+EpY0_uDLZSHPVn8`3z_aJwMRBvB}y< z{y9Q-N5Y8zUP-kTw@Ql~cPZDg+Iv!wEsa!f_yT66$C7k?dTqF+tq$F^R&_4?*>?Bw z`$W+r2P!f|j~brhNc)zIN0JnZVo}~M2ok!zH@bKK#oxxA7WW%ssKpV)cCG30{i6~G z&PU|Ep2;8U>2&C!2(k4yZf8y0C?s9nQaGINr1K#}XNE6NVgG?Rk9UY?roL;~FFnLs z&>~fMF#T11$9Lj;dsU{Z)T-Wp{h9A>p?Lm11m|2ck}7#l8^Bep-kBLBITnX6S=A6O zoGZNF?|klD)RPlzW))?}r_xI!t$~LH#4tr4j(2Rz&57O28xvZcYmH@fFMoY4ECco1 zm|^q6$FapCsV6vXLj^z{TYAcNbA;BURenvoK{MEeZNN>IV}2Ox12^S%f)4dZjX}|_ zk$dq)6an@b)f?vt6D^}Sfs5OkA1V&#@GhAM<}<6k)2d~TN7y|&5i@a-z4)GYa zgI)-nGMlq=6efCIIL~jTCYtHWk`Xc+hc^ecp?ITZB5epC47CVCMWx36k4G#y>}oR3 zJ@uDe$v!a$-DZp?m0dpI^09`(dyRIqCmp@)+>hJv!OlMoe#H26&C={DL?Kd3P~RT= z&>})Y=@l4(3re$EqBaJp!N~5fPIPFuZpnvjyD9Bua>R_k zp(-yaW7-O3iMh8s1*XwOrTn(*1aUHR==#uv*>WwuM?^W=ni`2FYHl#)crm8k_i?+J zx2q;ur_apeG{pTLmzh4AK4aa6W2X&F@CBE$w~wr1JeaC9t~icf9J*~RwvUIWSic7&zYRe>MUy-x;-hQm&=wt{L#1XE^|cn1BOM~|d>zDo z+ZWx7lVTa8L+t#m#)fs9^QasQU{Rdxgc(}0Y9bDgzD9>S=7}jAxX*rXXptSOC@ku~ zDf0kLOn9b)D&WG4o}__#)BFCb0?~`rE`V9OZ6Sv+|0-1hO*Cu@Q%Z-ZJ3=nm{SZNU zTc;y8PDYx54oRM%p-!S(jzos|&qwbf-IXl8L9L0t#>c(AXiy-B|Etgriuzm&YRv^b zX-0<}(n%gY@#yge;5*Z4RadQ>x=KnWZaOo+tgg$)Veu^^ z(CVt1LYiK(K>_!siz?uvYKE7=0CysJEujI)q(j>S=oS^qflVAB!9dcBG&{qCk>RAi zf=DMqR$M}QqzQu8_iDN9l_l-IC$eGdPxe0fAb9$;5!loc{03(1KAa?K{xKbXblhyl(*CGr4mbBl&RZ$1Yqog;Zs z!C?BMLp%9o}sk_%qk&Ep+DqKLHn80^oz$E%M>ThHHRcp!o)PsDz+)yj2zJzV; zFTpU{{~^~uO#OM8rki~>QV`qCE;6P;u0CU9yoaY2!TU~nh;TBDe25ZQovm0vN|<(b zjZ319;uZ-d{IoX%eJdhE?7)Am&!VnS!~eCpP*XY4qT3T7r~(|J3cU-1@5M-==AV#m zfd<*aQKHjH-Y);r=gcZyX(b+ilm;TQe`)>yqBz|DkK#{BwwKL`_a6&YO2@0w*fXBOr`?>dfqr0!Ag2mHd{;xT1 z0@n(q&Ah#WLXb#RQ+@7Qg5j4%_9FC|S z?n{S^vHeJrk^&{S5=}&YqC*@0(O$HE5)yCIYfa_I7hmWg*sXZ2l#&fd7-BYQKZZ+x z*K-G-oEDpkhIK5w($PQf$~lFDsBgW@gq(~kE?{Y_Nh=9eozRTG<^mksLc?N1I}&d) zk>awUMG8kZs?Z@)-8M7P0Zenwl#?W0!cx9o(vs_+$5vv&=aeuEb>0|d&7Dp9lDT-p3lanq94ttpZj(W&NBy?w#Bch1D9SW5gjZN$+&R=>4G7f z-5p$uV0&NAenCrJh(lc8O%&UiPLkVdN{>T{HYhKRXt%C$4wJIm`&+8Fcuj)e^vSPl z3Bh+i;IMlqrAf_=w4TI`Fw!Bft~XTOC@W1%f7KQ9^lzdnGkhhHtqr~6`9Usk>CkUQ z%i;uP;we&UY51Eq-PNoY_KRX(Uv^u{K6E;3IeW>_mp83;?Nf~51?q1K)IyYJPqHO` zj;Qr1WQ{AFG5FfGKA))-*|A|x(+CyBvQz2o&&0D^imqu~2fL%L5n71Bc$x@U zJ=YuHPTJF}7&pn4?1E<__csqk1zz_GIb41zj8EtCiA{aG=d|95nx1Bt;hJd%iVmS6 zuVie|v>l9_s1){n&ZoP~6Ag*Wv+_Ms4~F#cb~H8vyzNX;YpM8D`pw`{&k5VXl;%B` zRt6Oh#c`ytj^!IIE5g5{hXX8Mk=6=)hfw$vzNbHHTFLy-sW*&tLtORi^w(h7LVB5| zebv3s{J}^^8vYKOK!-p;ILbeRY_d*qRv>xc6{a~xv}5WIhHz%5x<0#guvB?PXi%P= zzh3=-VyJ7^ZQW38F$lLUkr;gU*cq!=i$?;)>&)_LlhKE97@~!H*K#w)q>&DBoM4{E z-N5-{M<+UJ0?(2Z+Qt3U3r{dNsV$)HIkq;M4Tc z?Fr&27^YzqGinh{vzu_B#etY!WN(=6TeL6iRk!SWyr`v)q7iaQM}OlHcXN6gnuWG^ zIJ}~*K?#R`dq|4FK+ zKR1ov$#7<6q3Ek-QAYJEC3AV!3UBGy(cjckW*eG-vhUHI9C!t4swEwIMy}h|10Fjz zh9V0>iw-*~rflP_6XlRpF;j-;S=(it_iK|o(NV*rRS6e=cF@=r%_xdk*o`3?Um>Vr zte5!(@PEEL86*^ddX)#T`l0C3_ z`@rI*2X>9)z)i-24n|N-3i0=)@pVIH=xZUt$qOO1CLg+0`l}jeLUm6tPqkxcZ*fat zb!2-_37HB15j8bYJV8^8z-`QYlR(Xb`{wkr{j!%bH|`AGLfClFg1n4tjlmlXc+X{vzv}h--HJF8{{YPmb4}q)fD7erS{| zMt6gmprg`dRrLpUrhvEw!})P1zHgBx#h*Athsv}*Q&avW0}-rkHtp~PHxAsk{7CIE zRor^YYy(5y8;K-Yyomnb!m4Rt+oUTwHa2u~!@@JyUu)1p?#}X)eXA?K)5}i%+^s&m z*(3-a?!&>_2x(6xSc42{jedvuaS_9eUOv%_-iQ~Yd7*sQ(s81ZO5|gamVFwk;=Y5S zTDjEa%S~Jr&szk{MF91;V@WPh0IzTzGOyUq1EJ9_;8JdKbJL;8XZ>L9v^ScCA_Pa_ zW4xJZQ(73xC0wd1=^}SJW&;%tRA8>d(jmElURpm&YW3J{Orq58V{Lml>Q;+ypB9Nd z5Nw7BV~RnQTa#>1j6FzvOn4-SxbB@U!cv2$YE>~>gR4L81U^(S#;>>tDzs)$wz1)l&C}|&~!=b7QHn2 z@MrHoO|HYS%(O-^J6cAE zO9%JLJ{a{$gG}rJI>f)m7oD5B0)@vvreaH-EsGa(o~$k|jN5zWFG!h1h`gz&sJ-x^ zwSvD>HnX$1UV#sl6g>a*kW605m00k${5xHVIT>8FY8;4g3?(h`6U{;0Q%=f-Q7Uc2 zx$O*=A}2DAf2^X|hyNUD>8R@;#_W1L5WptzlP#+@^O9PM8(d@|O%YDEzcHEKbYH`p zGLf(ii=;iHvs@c1F>&_V3CH|^-kuoZ6qnN;Ub9n&>Q=gV)|)TNj<$i9uU(WnbK zbxL?I?~(IBSFWtaVXy`%xtfyQ(Ls|jlD-#Pw_237M_f=^{E3cqhzNH1jU~woKO3b! z=tEH|w2-~s{-N6qw$I+wLQgjrT3tOJJZw!>51=yPl9AU@Kdwme0`i%D6f7XLy+!*8 zKVldCY(}X-QxQW16nG(<P2tu)l~riGP;&oN@xJcvaTrXOx@CzkOb3UOqPZGI_B- z%fllS!J8tLMGd1^)1eHXY0&B?9)fi*i*MLz%p`^aI>b)RwkCAw)2IVP8`h7aLoy19 zxMj&}fSftw60r%P#4WU7q?~^UpOrw(hpFZzq_ z79BeB7R`qO;mWsC5{lL=rbBfvz%Y1i zRRormu`|{*u^TgURPJ#Z;M!M9fPW0xz%HW+OEapsP(A_iby{y`qOPvC&p20DJj$^j z@Jv3ntJ7k4_Eg{9+v+THk7s0JPz4}cFH|`X;Ad{>&J2*R{HrH@WFmDD1Ynt^Bm~H9xx@9)X@sjKHNX0k-X?-STTA&0s9?GW38e? z3dQ&yTEi^pE~m#J^tpVn-*P9&mJVU3)%bCEgpr?ys zr@wLdKg-^&{L<6pW9#|%;-XD!$s3OWaW|p$mFlw%<}BUd;r? zEOLu3IdQkD@>29jPLpG{9pf7D;~mC*Pu9zd$otXC>cV=bE%fu7V^CSx5pm)J(k9vJ zYxpJZ#B4PdeUwIqG0{qWk;~I5%`s3b`rKsbCKkl5!|~(3=r7F8XKHxsZHjJ=*-M(A zZTotDnG)cg(rlT{Ko7*#>!P&g0LNqunLW)fLYl=WpcQPnIn_9ocDl z5c~v-zwr_G6P!2Z7nvoi`qCjGikd+&fhp_hp(v62+N~ZQXU@C5(~-T+WD*`g;qNCt zB0VHq&eJq-qzt09d|a3hXU&&K`&Bwu2G5LTUoiJv>Ja?7n|zekF;UfVgQ7XJmYBcy zPQp#)GHie(s>Unp%zMlD@VgF}vD?p$MjzOXBJrl0;gP}wnS-XHs+GObPd=&{AtfTi zz>bt}FtZEk*an)qI~nIihXT0M(0n`KW-dCVLAV1^SNidlzvuMy*lGdKQ7+<_o7y7S zR_l?3)@_9w=*cH_JxLU^D@1>7OM-JwkbaC;SGC{1Q|ub(tvAf4d6AIqS}|by%m}su`1>G{#im@Y4gcB{$g^gNs`&$S-iBKY@8Up@3pd;L3WAy``eBN-eko z=4{J*r<^0CTtA36Kj)R7bkOM=#=o-w*9HE04NLmTPFHMmbZdh7z4V_G12~E|1<`apZ{4GQr9lGDJ ze!T-sQ;0cg8cmDV4K~>g-p!#KZ$V!>+MWLbHK z@QRn7x%t>sl!2M#a9IrUAry^|?#Bq#yWk6m=LyI*EbBdC=TDVi0(Oy)c$!Ccu$e2e zyqin&Z9z|BFn~L@qDN3I;~E3e3O(o7y&Y17s&Q^NNB5+z%-sEMe=oB({UBDKR6srw zDM>9eL@eSe(2YI3ZDrwet+_`>M;tT)8{;0C_*lMQsyci>~K{ZvW2 z;Un}QCom#I4x9L_=FFwmM7Luzj}#Onf~RBlL|S{GNqaVy(?pPxK2IgSAvesUls0zL z*abbJ(D(yQYCgmt=XAp#9pr8Q;g1~Na+8&IOS!EZhD|b*q%}@duC;I{Ulne5Nvsws zu)xn21(Yo21e9og>+<)>RB|nt*u^_n;uWJr?3of?IZZa8_0@eIil>P!T_MF1o4-0+ z@{iOdPP@gq%|2?HD*`+qtN|81ima?z-o%d&P36SC^tZFDeEs^hb(N2HU{~Gu59!$~ z1>)%vc9rOsPbj}#bV%iD0F5;O)r|9u-bWk6m3M#;SEO?inN-(ZmR{Nxz}bJ8cx>i9 zMo&cjvF*d6nD8}cFtE7=sBwus6uIhibZEfPnx+kQ99-}l=uyB*y7B=HkG1TtJ*t9UXado| zSg;h}Culco@$j5u_=I?UGoNcO7S{`_DwQrY)|X2tb{^v{g`&OCO=6W}*2hTLna=|& zg)!AP++VmSn_nM3h+tT(mFC4aVV7kq{<7`=O8W>*#0 zjF9;g{}C5885^qT}?}))I!wSKgd3x#>AeX9U319U1V0CAO9XnK^Z6|qW=)=AMa@gk>B@)=zHep1jBI?aC` z(uIAhzc3rIdP(+m?VZAG(|sSmRUA<3l+AydnQ?7s>qsAHj@-`N02!aGneNjv;8VN=u>qWzmy(!QR9}hhf!oAE)jkC8d!imIgJ^C&$LR(eD5}L>hscsOzUE7FET5mDb)M zOW94bVs60-e03x#9Z5c0Su%UZN$ZPx*7bMNi9c90DIpYoDAF09c=9^6_zA71`dJLB z+8T*|mo%+PW_AX6qw zW13zslV+4JnG4-HK-mulQY@h6hcX(@lG(vN&sp5sG+=vC_GI3}($p8x!U@^xj;%uB zY*XnB#~aDXJ3Ai#ErUnfU@8+Rw~>iBA~b;8Fdq6b1ZFlczB30%zy-Plp%-4P#-Pfq z`O)uci|NqN%ZY6m_JhlH%#Tx)-5`Sk7#D@ue(eA2{nDvdKa0e`9@-?X6{ithqB%niv^slY)`FX8f;y9C0i)WLMU+e!WY7^xpfMscKDLaQN=b9=|IA zPeA}>E8rH^x(`m>1l!#6@NEuiF3KyFo9ICb0^7<%EhBGdYdixksamz+4{4F(zqV{0 zElHXC4cm$v>7cPWO=BfVj?>nS;y9*-Q1IJV;ceAXrpJ38$W^r%&;B&!srA3#P6--H z{$U3;HYSUgGyDP)O^6R`FEh>bD@0$(e~lf_CTWrnlDry)XxgL#5R-8)JQ;6Sdgwqt zSsABwY`#<@^WDsvuZ@Rdb9Pa)7a(BC*i`xnw&{aOj5wA#^=G{m4lb$g30!2_%iVJuDop+QpLb` z)0X%wch0^Xzk$J9v!;wHWYYNL@#03Twme$R^N-nkziYeC6p3KV40$M=6(EA}T;Xp6 zFEI%UPP8TT)swqMmrh05SLyWr4Ad`Ozkm8wRmJTq$m%crr=|De5CLVi3Hj`*zgv+G* zi96^4cV`bn^m|OT-3`r$s_ir;ylS#W>Cu4*VomnteSr#jqE7F|y~OV%Zu7g8qROvz z7S88_q1kjqj`ew0m@sAc1pYy#)vhG}^&q14{_^W@LaFsLjBk3Gr#e=63Aav=RtPSw z7=cU6rNPA^mWmz{n_I4#NwsUM)-@=5D_>N^ z&QLQH==dZq^TSuk-M2F{`ysct=wrFqM$B+cy9>J}k}pypubNPGE6UdLb+ZlzBq)4fd7V z(zGqAT^9~@ZK5=kwwHj8kK}$d9$wI{TiB3Vu&~rHy}EcvKuy8w<4PkEP{9kROPDW9 zEvWMRlF54%h3QngbF>Mv{XLgTw}9JmIo?Spf_mpP$*E*z-}P`yq9C3-Y4W;frUgFg z?a0%cFAm0k(cv(Xg{M8T`>sF{teIhMK_=Ny3@}qew=}^1XjHO+aknpf)&o=bl$&2q zJPnVrGPHc87 zqyKMwM3HDh$U1~n2-%m6l08(CEk?`5`bD#U1_xrUSXt^vNw}P5&?4BKp@W-6CA_vRJ8i(Jt zmbvz`&RL2;n=X|tCm@)VNt#~<0ZSM-)! z)QAGuiN7)DDb?O=@H~130B|Ojz+u9ggxzi#u5SMBq7WBXii~F`1X$|tL^CdZl}ZSW zOSZ+-O5(q-$j%7wwwL{0tL$j;8tY zZqy>EmI#TjW>FN9y;Bjo@Mxhf=CWK2_yabgZ7CTRk0jhHxwcL!ZX%ehZ*O=aE?JWU zVY=V7!ba@#ij(zPY>ae5i`!}$da>hA8M)g3X&sU0;fSd!T-a30=~2ZkFTo$_ic@mW z%zHIC77_S7gT-E z1+H55>8@{)EfolrBoD$r+U#itt2JHNW}2!y)})f+>%_n5{mkw3bl-Z&Tb^h0p9%2weXcYHT+{G*N zg?9`IH|s(_UB3hg`wObfY$U$YBi1w{k@?nb-a^&AZ?tW+>zTevt!C|7{)A$;yb^%o z`P?-zGLcpWHy@=GQB?85>2jsOhM=nD(@z*6a`}G=i1ny&-lZ^ z4#@*djO4xpD=0!JS|6_*8D{y)$AQ)*L0PA1Z@@!bADd z>o_*cg(&*z?*s}7ZirC(ovAWa-K^8fF<3HZZ6ET%cHYE=OC2kw&+_7elc9Oy=luuB zrTBI)@*du)eK)GzhPyIBom|)drEj5VeqJvMvB1V^dHmKeV!r{)0Y5 z(L^{iINS~be`a$mP|F4n?@ou4b^-UP+o~BypraKb4;8gMqBt!(X=_{WX`|yUdiRat z`M_2_$#j}5vzQX~H?oaK1rpN&1!o z?g$iQqm~0X%^^yZhsa)?vA>{g0SG-Isl`cXD-SYA1;lM)MlA)~3ux*qT{^HRU!|)< zyZtqJbb+=HgHq-z)X{dZE9h1+}zwB(3A!= z$BlZK4%GEeVMj`4;W^Co+ccmKGkAjbO6!&#qB=~betPg4!a z==>SmjY;)M^vUXC_w)C#uR{1QinJGgHR9G51Z6W!TVTjGxC_YN&=TZQ$6bIS8jpq6 zV#s=$0061|$&AX3`U{HAp>v?vS0OXXS!6JvMc{&UN>XLK;-CkRNetofHzaWry6g^r z{1=o>2YyR9I{;6h?lS+ml$d)t@z$q%FR~dz|Isuc^vW4hY!4Z?`uvbSH+2K^ODw<| zN~osGp^i{nFtXgZ26FD%{-@lDogJ5PC8l&k@qM8+|CbUUI0pvilMH*s*FGotCTm0H zQ2=jqk$&PIZ<7X8w~14EJC33HYC=VWTBiZ9`T_w=3spqZBoZ)*L$hKlE}cRut5cmO z*Rh=q#pfb3;_ON^ z=6rl7$)H!OnrgKc*uEc@3GOtLfmA(G(h1+ZjM zoj)0u3t~~F$={G0nkm!=a4vu`X2q{Zt^7EKPT)ed2P$o^>nyJ`&tAdhsqD7Xt5C%(3IJZVO|tpuju(Eo4K7(@po z6w4O~INAvhC_xxXzm&xr<;NmnxHKk{lXRWW+CkSr{}#)ma9Ah!zx~($*Bd~xqv$)e zFi>?LAqN=m0U}aUJggr^<#MN+nJ;J2LI(dsbsYKpS%GzB-1-3^<6c(jXWqN6>9=6~ zogRm=R^=IJz2Sh}1FAV?nNnao^#DgYuSjZ7O$AC9GxK|_`&?!`UL&edW7D%}!N8x( z=BIX%`2d>JtQ~r|hdfBPp{f6mmlp*f>E37x{S4K3Uw^X#;6Ds7e)lJtH{$+I60&CD zt8~^!1zAUYnkr+oW7;Y$AL<$+KyoH~LpnnOj2b3AY>sEKJ=Ol!D)TiH4v$`k2FFb; zVe!V&a>zELrkPx;3Ap}pg>}NP95u-d1h%08>ql78*0Ip;cC8=}e?-;(HB4Sc`9Q+F zVbY6SqBg{lK7g!x7)>#St3WyZ*e`AAR`XWQ1o3*7T%x?3#a=dA$_iefeWjnbhkKK+ ztPDl)@V^PR!$y0s7up%~rCjW1toxcG`5LQ4&4Y8%8W2!8Gr@%_k_E1wF82H4t2C7+ zk^W=xDb_dR=Tz9MCm0jt)1g@$sh=EA(I$i2G3PQjy;!$YxXz5u3Uz&Di6|b;;ahkK zvb%dWs3-gEpSHXDK!=)FkqHqpmh3!5PVvjs9>LSQWzyRPl@BxZ7wYwV1WE(D5MQWj zN9q*ftU8bA-9=0Yt-j#)y_L@_0&_upN~74*uUh}niX%HizTA()Qp=O zU%&C8zE1AsbqBRa*Vk*@0dZ^xj)5Go+?>{tb=n3CiSQ_4&uHT4d3w{c<0Ru+a@$rq z=m<)(qBU!ZQOpAy{MCal7}G_#L!hoW-rT(}&$bXe^E_EeH^6O>9Yv z7e>!JF0LF*;b7GP9k`S9HXQ3J`8!3%g+hjWGwKG6wFdzgM6n(M9 zbFsK;r2~evXKhzKSc<`tso1;_lP|z)pow&3T(*;aI0PZj<)+TB zy7g{gm596eS1W@mq_bmiB{zylzti%!COf5$P4C-?c?owbq~EQ|qyk1SrnAA$8Bvah z`pYWelC%s!{8e(MEj?p&>w2Jo?zvvYtB-k4jA|wocE#+RX%%0=C}`>vqizHDnoma3 z&m&(y-+?m+eXT~nwcmdj@41|G`O~vbu}Iw~R-cpW975cCv^e2Lz_+szV=cT5O@xqz zH(Gq-4oS@8O2&^Jy{`8(T01y61uW4HO)}%2{VUhXvhWnjjjbt!1lg=8(A)&mTy1+L zGvtcHJAd%XlX%ZOPg}%7 z!j8{by#POSKX{Ij&T6s*i-9mxk%TAMHudI+!~Vhi*xpR86UH8DdQZ4x9es%1?uV-+ zZ$SB#M^#tOszMaxZz8U#_-3h?JaM30!M;~}pvI`SN>ti|;DLgJX_m~lvKvXi0%4(w zIy$W`+*Pazs8#X3Dl?G88}PE9;A6I+Sbw2D&G*Q$F=^kTKIlut2ba+3&y3S}?hk~f z=MQx+>)+L5IgX(NY~fR2U6xpX()@RvwTE z75qs;_b%xz`E)5;_>%4oH<{QHh;Y>7JYH9dhOk3{14y7mUA?Kn#axVVzY$dGD~JLm-gu#K^~=V z=+4pJPhMxg2@=gu(Oz56SAP{lYigQ2h?mX{PT5wDqkc=(y)~`5>Mh863#y=X9ocqY z3qs{3=d74T%wk_3#_R2NSYDho{vc7qVT9*2-ivyH^T*!%4@Hqiwwi_zN)P9g$9@3Y z>}tvpj1m;c*wrFJ%cox=qYOrNcMj^6YHJss&UU>vi%q%bQxBDtrZP zV_n;p*#4C2kp3wLq61Y{kgYf#IO`akuV{9YQ_3ytdE(bsNaa2qt{U!G59!a|nR@Q~ zxFs}x0?=`D6HZXo2&XV!j=9!K6QepZ>3zln-G--VTYo-e&RZUEdr`pjj^G0hcblO| z#l~3LP0p1Sy%vul#g13yof$LHiVXYfRE}9Qg=I{qo6(26H7g<>3{Bu8GAa`-bl|v>vTz3wqlV(TINN34|1VpaF#kuL3IVuEM}S zwuJH7uEnoU6Rme!2J4K@DG&P;wO(lL0>J}-nQxOk01!977%vcLzzTy?kfW#2c+RF! zHvx6qEi4g+%?==tGLc*gT*p%f57qCU7+H+b7-n9t#RH9t1Q^i8ls!its~a7?4?ySt z-(8Pi&JiXSy5?p%PWw;`)OLYUlrz)_%Rs-dwL!s?`}>}!KU01c&mpl*|8#HM)M&)q z;#Hy&3hzO@K4mhwBa?0NQrETni#UUMZ}`{Kch3g+Fz=PVMCy%^@6mkoB0nr~*Srn8 zuB8jWq)3t{ut;l_BXMWw+)G&%kH&^5*We=c^#Q5YaIr6qjY$Iolk%K^)+;!9u_XeB zWlYyiL000}MHZ*sQbG7GP zN_owd{g~42peLZVqcIg!Oy27Dap%vO2~1cnp^td~Wb%~_eI9;y4z5v{*v6A42ndyx zv)N(QHrMfP_g)K9JE+eJ62BhomHY*XqnAslUaKEQ&sPvn4!swR;+v4T4^C~2)Ebr# zPmE*p>w1~RM)g<)s4=pQ1|ISAO9;&L;l1yDl-CV!&sh#V4^zklRH~#Oig;Ckngw{l zc0ePkIPDgjG&vlq{*#w__1896xBI%;?B32VEXjsk9pbusof}`8!{Q|R zn#n~Z>@wqGQ#>y{$d+!XpZFDESu~KnQe@k#6x=V(v-5(2G+)7sEo;Mz4Z_Y@cf61xuZztJHlF345vO9sK|tM^rU@$p^Dx?FR#gi8OYB}VlKF)+C@=`HUQNDtcwod zFi4e3Ybp_kl90yHSOV*rLr>cpiuq( z62r~uR;julPh<A@T;^>2M`uSFOK7y*H7Y9q;5jg%gISgu9@a}%55i4 zWsl2q*a0)a^)J}<_yQ`A{wjVTM&Kr)Q|g_HL9N5nL(Q^{#TFV_gK`-zN6QB`VrKd@ zLd~6G#z+2awN=^X@`5;>*QqLB*lBoV>ALBw%?n*BNzmUO%YQ|VHCaUy`H#S)6SR6Q zMmQ@i>kC1$lkxkt;`9uBuWg93`v+?M+O-dUdXJV41`$}qFkt(5{U7Q7$+jg~;obn@ zPwexSp)-At(*~nN1;|>t19a z_;;ge)kQN$$}OH5z9&u_vvG-Ce1rY2MD`~X7{snbb(yh|_3cx=%G~*@Q)1ukdF@>j zpA#|Jo|d>N=*Tzpj5IU@0M{zvS{LclvtDg>!M4=|+GKO_#*fDza_J^N^8c z$@9*bK&m9L8`jc3Q@K|{#JXlBe$;qX+1AO5a>5kfI@N~!=Cch`FyZF6i{uHeUV5<|8F1^=RepM@JW%lEzHhm=dawcZ?%)%gW%*;DO3Jx@N^@i<|mSYL6frmVB$sL+>-8 zOBOh{X#bQ2Z#Ch410?BLo>ReNcq;?re(AdYm}BaazL+$tUVtyEy7yPcQ5=1F0*|*}EVvT`RweNgD;V;n-RLoZ_!P=>J4QtmS9MK(Nwee{ z^t&dT?wn$**H>JR5^->pOolPXf0n%0CjhaIjLSJ=K&v2^6D8ZBC#lSZRAXXbU{wgG zyi7;i=~P`kQ6FwSQ-vQ=doa@L;e6mVO}&OHMf~9X&W(93{A=tk|LCJ>m5c!Xs#jdB z>#RCYi9_7Sc|_jj(g@<#GWSp@+Z>=N3hr5xDc!2KRMG0T_B1>eH zrhTO{Rz^LnA+DwI*5c8|O8vR1-i4B>Jvt8G+I|FEik2UP^T~`>WG4!r;Sz+i%6yBT zIl`r?K2^Fwv%IzKgMw2+@SY+1C*T+Gq@}=vDEG;Y_X$H`lI39`Qg^DG-r;)1UGI(D z_1HU-6q)*+)iFfAg6sA|A!J6iz=6p+!zEQHzJ-n)d*f$mQZL>u?&!Pr*DSR8#;vFS zLB%hE2(?tsKON~VU0I<#=oPK?uUs)b* zP`IhyL>OL@z=~aoiuaItt|UMiyfxnCP+C^}a7@uS{^uuEK1pck1`N8HMKFTQ_^Hv#e%x|E&8Q-zrm#kgpMNa?Ce zQpNANt~|7EnIQ#4NXT|P9~#hJMLz>=0;@_Eq+F#7HRF!ek|gqWN@ZMM8M7Moc>aWP ze$=4dN_^h<)V((5+BK=eLF+=vx~}(y%Oh>kXMt&UwiQ&>o9$Vh9$oVJtM{;C)5SG-R_>#e{%Z2 zDrM>y#MBvLpX_Y%$|D3{Y$s-VwUal~YzA5tQ5P#kFY2K!&hJn#eV82F%&8mJNGDvx zYUdf%actCcEkMD#g4E~}pbjbgT4*fY@wKPWFaXRldb(!;~UQ&W-fx&w_30P*Q!b%;)1N zVzdWAf2CxA!_Jm$`!MjQg`d;t?lkw~cQ>tAM8e!ww`F`shxF$hrAN%%DlSiPXlr8<^o31g+Tvjb)#MvXw(hY#k5h%1@960#-b+lK zlzZ?U(C0I8B-*FZn!hT8#$|fNk^N5(kB6C4s0zgOu;(p;Wcwexo$7uHZFw`|iOus@ zfA_P773b}-Xr2qU^#13bZBZKHIbPL9a~goL6$rqC6$SJ+jO zr&cvB=UMP%0z$yJl^PG1&394z*Z!~X`e_&ZQ1X{HyAf7RKr6D=84Z8dhc*4f>#^~6*0 zWB~_9MFvwaFI6GqYYe-)e&Wxo-->*n8!Nm!f0?~kJXC(Nkt2U&p2LHeM!+n?x-qRo zZvcP1A0UJhW*A4q#R_fbbV^*M2aZkVpNI+J%Ts3cLa&B0raqbO4*G?uytu(%m2|GK zUOW@TI6llAj+kHsUXMr@aYeTw(vmArI<8BsMwMh&P_7{8W@+&53)VCKFh7B1028mC3 z1rg9I!dY%wUy#f3_h)Bkm6NO`${Gwt&O3cv_{y4n)=(Uzo_+|M;ia2)Ec4Rh)g|fg zv9L+NFYfLV^o`z+;i;mjpT1UF+9XvBIHp8{#1rx27Pl>!D&MTb!s#VTqEIPZh@9aiLn>E^ICL#Z%>9^Ya_xxwu2b|A!bwIUAMHbQ9g^CM@`br;O&%|O{O%#L^h2j*o`!7}9d2zqQHgUAT{ zMAsbzE(5^yzsuDBE+A;uTXrr1c_J{s%Os6Vp3RF5NHhT+*M5>#DU z=IXH4IdZt}cy-Oh!4dV7fVbev1>AM6y(_G|-_TEKMaZSR(*&;-NF0K>cS&!;r1->) z7Z1^K-g8+30UT=MJ9b0rsAl}@q>V;kDL%KRRyPLn(ZH@xK2zUZTZlIj2=3G)y!_yY zVkf6rj7d|w-*vfvTbRW*H`J!FeXE!J-W{^_x~aK-+%vzq<>Pwh@~ok--nH$W00jkk zAi6?4({OESD{UKCl$a)mp>9KOpo9v4u;Bdf`x$zhJ#)H_=eJ^=X9ZrBsd?6Q0jV(B z1mi#*Z2E4vw6n*}stf>$F^ea8xSjmG%i@uvS!Obg*M<}$3C2V5(8{#-1upDes{3SB z#?@CF#q8PRPtMvohClywy=0`};8z_?i0TdFrqAD%ZTGr>Gh1rJaQ^Oh|! zvns*ciIm%9N7D3gi~QE&narWKb^FOf2%e{|<|#$ufH&;4vUL6(X!-|%0JgnT>>=?K zWMFKkx=J^4W)Z75f%c<>=ybXZ4ZN(qKbQMkNZ{#L<-1$&RG)`kXK7O+F?aoQZ4h#h z&XlZ}RZ}|f>c8>TudY*`qhjQ5yRxwVP~Cu?f#K(aS;XW{_V85YoWCHE63bEQm8CPXXC9G@x)D-Q zw`wXRf9%(Up7(#AHQKztvK}ew#qG|IPskB!{@Kx)bla$0z>&dlNQ7e%BL#gfD@2Cf zB76p{wdTCA+y%mKX){NnIBvO zpCI$zCk&!uo4_mInsf`EdGA)cTkOZhe}2j|#Q0?}c=rvRQD!4->uy7$3~QwSm5N&n z-Pd18?dOn~h*T{I!Sws%WhJTLG`6v$)}Y7v8nt>FF5eF|r+y=`s)4iL6<+ zzYL}~^HZnx8dB$Vl(#ASqs{z7*i7k0M$@rw{50s~JkET>-UK0B*t}u9dxKw%293D= zXAt>t%ulyv(%(9wIVB>la_GvM{&|N=d8`A25|xSodgzmgOY|8i?@%{-CPzn0gJ97e z?v)P=iy70s8w=&1O)gk3D);f;HI*M?7Fi`_2by=bNDnueEh9O9kP_X~O}dhdv>kar z$HguCDAd06#>Y@wKapZ*M z2Ju87nuQI03VauH%riBy+sgH;FgEyQvQMToIUJF$mH2h`CaFLArqtO93yrX_U}CG% zzaYxChd^3SI7HwQK@2WMv@3`{_33POE!0VR@m`9y_6(x{%80f|o*=Y6z1S(Jeb?!g9O_$dm9E=6`R(Ap1u{t>wXc1}M)ELujaRzBzg z^m5$iNJ&9D_vTuQcUYKjz)s7>p6H!rVcWypAKa%s6jehX%f z82Jl2^)D)Ak$cP^iAuHUszLc8*K9KMtwMC!aVcWdZ=RQC;nm12(5hH7oI*f$?^C77 zS<6Gb{RK+f4QC77gFSr9bLF`9Jh4pcC5R1Vdsihem?gYqnL@W(C_8W4PNt}Bt7A)0 zf9~X0F+R~-iDH~HJ=hbWP*$oa(R@WR(l5G7_H1C5<>>R}1h@NAH;i^>C45dxi^W*n zSx?^E3;u<)nrn*~QH|$lrnF$n1hcq~^26aPu=!uAR^G-f-#(FnKqs_8uyc(ju#F@L?YJG+8Re-QUV0 zME3jmGu!>iU-qzyj_>e%-^0Q@AZ@F;u0))-=l)0bRBbqR^G;a;qtI$@miP9Ffj^^P z8lgi$gGb~N1L&lrn)uSHAha7g*Q>=C=-34*pGTZaw$TzOOYXS0+5Yq5e!tB(uAmz? zg&idH^uJ`YAeY`{0x^VfEmz3Da9s$I@#+tchozkoG`6-rRT+wZ*mdISSvDkfA!wId z(d+JXmM-=QOjj19WL$<39ktDHRxU8(z5ihs@^(iaNJi*)PBRGc3NewjclQ8@J3560 z;!<+aAN7Ne-ATnjaS>Gj==hbPG$9FipT?9C^b@&~u9p&GHuDKd{uA}D|D&cOW^q8i z#1^hvwsX+7Ba?j`h=s2HZm;oX#IrOfg-tH>{)3uCgL#23Tj}%)eAKa~J}rW7@!Ee? zmc{y!v0G5-^i56iBCN!?(PqatC)sx)@xGPE%31BGWk>7qPA%aHPrn<^ZO_r$Yx2fS zq$_OSwkB-UE?Syz_kpW}l@kGL>9yK?n=N(AaQ`#?Qq!GfgJ;8irSFURJx$Wk(>I?k zI`@hn_;GU&Nu#@@Ci2GscXczSOz}pi#0%H2Q$L8lZWBJFsnNUbr0SFO$=s$!wZ!0O|Ij^FQjn%rhHEm`q9J?7i-$!ZWd8!g)~qGSek zw`@?ZbeBoNzpaaLb%|e2Rk51p4qRj1)XxEh=puu80v^_q%I&T1q@BC-Wz=9=T|G0x zijzg_D+c8?$J;8bK8}1g6sw*W73Ht-$=aXx04x1cg;OlDFRZjYMI`g+pCxq?*`A1s zSUkH1xFZBifj-rAWU`XH+LmbGm<%h^*Sp#`^)8Ihltm1AH}GOb3y&0*fNkgK$>URu zS3Rf4a&ZyxFzdjuwOBI-_|1B9*J{uK>Wuy1nXTFWpm4*~yOioPVa1Rhht}_p&KUqA zvvr)1zZzsUs*}5tXFrFGvUd1&*=e4A8~bvvKumC56ek6Udjd z*8~kn+ZWSdEvsToy2+%h5&kG1)4m&hZct`mk#|B{{*LNcX=uM*tM^aPDYIcq(kfxu zP74xf2l1)ccnJbJ-*cA$p->Xp9hf6CK#F*E-mgX{CepE8lA*xD?YOvi-<*Jn@*wV9 zup>7$l#E-F?Y1-PI-a1R!>V_$x^H1``Gwrxv8+pc|I?#K2VSwsp{$r}px6_>KUvuG6wQ|*@1_sl4w%r-!N1Ez*LT+bBY3sGAE zSesysS>z8mdaW-&xE%VfqECgy+yipTd9oV&ypBSmN^yU-$OFw}A!2^eS`xN3y>L$L z$p->{ecoB(1p-SORV%%rB}Bb9yF@l3m)pzd7*C1OGqxy^Bi0TCTI`QStAyE$T?r>% zU-;oWCImE?u~|#J-Jh&}w$u|m>xx*b5_WSoMIPUDmbRI@(CuR;+;ks)z9DsW1^ebn zT_hyUden+LQbL5*GXM1YH_h;^9wh+r(46mc>3T;sA-6`+g6e&0B`mel@}Qz>R=AtNJQuCVZY640aqWaEsm zs$)=I$lK6L9#s&~ixEKRtHkyACfDiLRs(#`B!BVZ#f4j4{kK{{jK0aOLK}27 zpqaX+$-Q#M)gP~AEk3r(eb4iY^ONB^(BGGMFNRy#Y@`sQrDe_eFk%Pt{PqeJ2Qeg+%-R`|!@g&=Z!moOK zGtsIHHzmuIwrc2#gpi-@Ha1%|3WEwv2CEIx3)2UzU!uiHCmKQ-_y8}~5a|(Fl<0no zmfynd9$XN9SmZ7R<@!BpBAu@HMbP`}BW(S`;(~=maOx9~VA3fQ7iT(B29f0frI*XG!a$F2(_(lPnQ}yd(N9f;?qd8LDh#!gr z*``4I-+a>ik$?`Kqqon3zaPCuBkMIzDST6i{~j29cpT!gPbpj<-s z9U1(s0_aLF+V6G;{M@+TVF@x8Loh?X1+?RsXBaZfG5s#PquMUZ$ZInSmv-YfrkbG2K6WsLRHYYC-wkfzn} zIEc-GSs12vw;jzzH6-kA&$#708xD-&TOYnN+Qm5=Qec#DJ0)B??xo7ibOB!*A8<@8 zSd><-d6FnV@-HV0Aj|$lT+@`HRV(64BqI-g=W-Tbp(J0Gn|p^1Zv%vO2~<9!F`f0w z_dtx!Sc{l!3^-!P3;$#SF_G8Cj+qvJFVmZ@?ek4cQru6tnj%jzrcbt*luKA9NxKBvRc>PSKee2 zzibGiv4h|MR2%35xQkh}gfG=nm^aj4eq3$*L^1H3Yp)>;1m}!v7TK^P-tUL zoT2yO7@q?QNof=oTB9GjOoVivZ>q0UQK!9ee_11M(7h-p2hwFT!1@HQrBgj=VMw>X zpeVpP`=>MBtV>v7boW}xp$zBjh^6Ukew}NpqRxEwEMk)1(Ub3isfeKuRa|kUDy&!v ze;qUcFqGen>)^=Di?r`>6XGT~3RfkQS)|ei1R0r5(uG{iBH6_^UgWiwa~f%@o-Hd} z=i9L@L6pF+RnbygcqF<}gc!54fqq+DlAlgxNXtk6f?PdOh;ZIk9Yok7#({41J}w-_ zM`s8eT1G`_YOI_IF@seq6Ab-bi&*PLe)c@kK4zu9p=DOHNYm8=#mvV`mMbf3rw(dm z#ZRBRy&o#xGOVvk^?p#p4+UPr=tQbU+RC+B-nZ3}W3svKzP;vrC*)CS-w)C2jijoH zLBN9y8Z!ilxUT^{zx|Z@N8`3+hps?nuf}= zFQao3$viDM= zS$&g2y4BOEuV|^7x>}!>D8CN_VcV%w@q&(Sl7P%WIQ!|;Ogb#qZ${oXU$JmxuMs%H z_uvQA=#LNoNQzYbv%vYAkqDgbFr~I*%ARXt(#mO)b8$thu!H5sb+cf6v1c{<|2pMBC+Ax0xSh(pM^~z4!xzH<_ayT5A1$ zHj5$8%GW>&)!K(Anpx|d?r)lpWICy-TJLQl+T9SBYg?L{ni^6>!zOzGo3elyqeiTX zXUfvz4FYo8oM>yUB}I;PS%w0lQ_pv_!o#I>*j2fgA}`fi7rNlpginuRc9+(>(dUe| z8|=`LJ2JCw7mjXuUA+H*XI0E+UAUS$QfPm%xoXm=JgC}Ct z9De^imTgp1f1#1z`&^_BreX>O^^h?meW@635fgev=kBSS7=>0_VYxrn=mGwS&cZKm zr$LVrGbs&I8~WYMQvBHyvpe7o}r5`7|*KlwV|Ee+Ub4DZcC*h$6LR&}6z;k|0pF*C+JY#ohGhSxrr}tKgHycMx z$Rq{?_3mvmm8o7m>prM)W>Jauf%`A$d9*(i4A%(K;y||Rx?G*V>3lYpmb}(2A=);# z79GITiZxHC%#kt)z^mKB5ooM82?RRK-blz;J+pA~#;*H3Cne(Rdr|KTsGW0iL25VL zpq`i+OOiW5cDrO1b|H{=c``Ql+2q5KUXLKRXNmp7R`9+}jTkE61`ey~X9Da_dA~Z7 zK5A;@WlP6&SvL_baGt)fh1Img1rOnkfnP_$699y^BB0}wj9CX=gs7EKA+_*u;o$^N zFGt>wsl~zYy(Nd^7|<`wQqrO-LFG*oBavqcHub!!AjF4JFn}xf_Sc!#+|rfn3`~}7 zS?5t5Lwv}#+g>?D@kxGkq(7MN_BFOrl)Fr&&20g1O5T}k;O8;N^2ntGD0fq@699o> zJAOQUg7tqhG&SZ2D7C#?G=Y29ry3Ew!mD@r7Xuq_rOI&jtsJcytG%z7`{Gmd(I{@U zVO25@gjk02k-aI`0%h&SWwn?#gdt=8*4cYI2?-ZD6$RQDsF>x2IG4Do(V{=PNOdGZ zm)N*6)OpHv)iUHnXVTkmT5dB9mjw@oMloMIFFd--_?=xwbOo>~0ohT2SLr2$#BTOq zO(zgj_ZYC8L=;dq06ySfuCyt3!|pQOYh~z&-{KJJj7P4{%KqU!vILZANct|Rmi&3- z>u$(P2h%R`*C$(;#pdb$G?dN%%3vrU9Rh~!c5>{0ORF2QLAJ@pG;~GO6cvzR7WLIl z@s|J-eB*{#;F>^6;%e$JXvYE95`=sj(y?IOa9e|_4Q>_31|4v2JUq7@=tyii&av?4fi<%YoW?%Qg1WhCb# zGwyM9r_Qo|4%ePU@iUS4HA?l@v~{yarnWtRO!+JVlHO$kC@&4g(8y%F5z&J##7iZ=WfajF9X+C+!|mO%gh1!I%(Dm(?bh-y z(Uk6+kcQtk{}VXI$<9mAzHSBgT#Vi>`w$Kt;r{@8v z?yAMoasAYRkkOXG2T0hER2%2O=#D{XdD1Ck@;3E$yq!wH8$l z+RbRFd;AFb#z^U^cbKf;Ry!$hWJ#zdvb-j0JQQDIf4HAT2}#A2xkG0uqzC*DR!hp0DUmmAs{??y28x0@gJ&UDL`3EH4FLS5ns|Hoh6EE0g%f@He*d{&(U6pL zweKeL1)#?i8D=IKWr_~py5=wC8BVt?D46^nIGEBTR8sLd{IQ=t?dTa@0dNBW8a20f zNjzk8bet?7VV@21=*aE{B>=q{={1WYd5%*>Wh4A>(R6`^Ny#$h#F?(|bIl#|P4@TO z%cWAUE4~rAd)+GZRA4rE4Ye$#1Mc=>ykfO_dqPq2$HXJ98|BVFG}e-Qw_tP3(Motu zGw`N2_S;;gQt=YQ^{2^vt%6g;ltTIdr0T<9HbJESC-&PogI3Qld{`?D;cIEr%D* zgg3qrV)uod^b&6*mn>f>6@BCQuMMq@W*MZ_^;`|z1Fx`yf&sc17Y=y$6mbAmIUx%d zDzcqxD{;LxaICwAxR*eh?!${|b8EBy{7@sG8HC=VtneV0K|~c=xa>cJmcyx__3yin zkezb>K9&8?r#e6w$3BJq-`ZT;{!voCJJG#{sq*9_uuQK+-R{;RrkuDU#0=T{&zuHu z(Rd_F`q?OSU~;FjI&Y-k(V?B`(U-`@$yqLM>Cx+9-^xJ&$q66u_WdP@Axsr4>L7OE zl?~agWZLZ*Zg!)q^Z0bag(q>IE&=k(7W(3uCqYQ}ds_|C71H-$bB*<^g%Y<6fdhp1 z+}n`PFM_x8h*9(|89*Ot@DsiE3Wzw;%gFhM&Yn3$am`!1HP{0uQ(XG>>sMfQcC3cZsf-iJnIE#x0`a)y^;JOmie$nptCC9;PnTqv z?u=ix{Uo~9uC4P^L*az}jaO6q{-xwCEaoKCZTB49fe=!u6SBO^xm7D?a^rpS-0K^I zowlELq^=mA)$*RESWW}rwaG0HW>P{~xKXO@g}HW8*TY5qXr{v}l5>n3_j~DWv^Co(wWZbuu4(bkYgxDMnkz#3ObZJC?rZiDy8%rqM zH>)N{My#TY#5%h#U!&LjFOkSu^(00a}iTYlX!K^OGl3+Av(jKx{|N>~tCBUb>jlOsCEdPlFr1 z8`{Ebqt!nLq*O#$RxZ#tLvs$$I4`m!c3X*}PW7W@T_jF*+MRuce|)I1nSpJt6>d4kc%W1kM8NGyy1f~c=id^x_7I}7q^gT$i_e|f`nb4bis|nyAH{KJVqdpS8op=N|{LN zjKh1@n&KYa5$yu zWpQ~06aG~(z8^mBdq63LXCQQR0t)*B9KWCO%r8O_8D%;S|5KA9PPx(Cpf46A{bi4p zP}hrxeTP;dyTD8UbfJq7mTU*lc>dlJmKfQ^`+s_SC>3~!U= zmc|k2vNrjH@+ZxjHLRe(&FDU%+y%xsG&k^g7~yQh-E9it6f_FQB~wyz_-1^!v~TXt zHL<;dkRF}T>)v?%P>hG{3~VK!OBDeon+AEp&HVlSUCVlN1_kxpwSPIdxKhhbb+6T& zZWT<-5Vg7YF!rL~s%-SXz{g2wS--su1uPWE9@1(tk2Ne^oyd|a zELksw#8XGCApY+2;Xvf1GlBl*lwO8M*6~W#GHeL*Pdmqi14=7dcnA@bK;_MKle{?z zWOonrBU1Oy+q!;Hi@%FMdPlT4EX*3{A)TX}|3e(sr~w9IWyA=z*`uba)& z0F(=RdzU{_tiSs9uIB)gqu0RePe`I58zdx+ema`MA1-dm&)%mP)AoA2;NjI(W^$Y0 z3wg3-B=CULkdCqmkV#itEQ>fFSVWW6 z9>c<=_;&t( zZGCw-lz-gr*s^4avM-~M3aJpX3~fl3$R1Nkwvc3J#*%%DLJ=cNc9Xrbj9n#U&pHbw zG0dPb%w1j!1*Um-~j zKGh_-u9`*8WJ7R#UPb=%&e@k5u^rw_{vTI7hma@^eRvbj)a z81MpvGK9BbK$dg&dxoqZ?2*1gNoSy*HhL}-IV~*en6 zLG-{u6%LGZoG3mMOeoz9Q;6*oY0Z1HUF@m0cq%=#(0DX2vw<3&nnk=;JDta# z^@`qLw--M8HMDzvv=(9N;=F*pqc=5~pJ_nmW-lP>7|nt6u0pO2<+ zB+^BK*mi{(qF`%m71@1k2h>m2gP_-D0U2#P0sXu0Uk@YgVU=Sn+X(~UuVX4xlL&2Q zewXv6WMhW!o^W^b=`r$+wh+7L0@495kZEBelwn)zzyw|RK(6UllWp?bZ7n8G_kHh6 ze~U@Xa3Ma=0s@^iRWIY<=q{9T zJig7xblJUo_~gN=$y7w>Tl=Lsoonhxy>iBejwSWVl^FB9*F9p;AAoI?iL0SHQT(Y+ zc$A>SQ2C!v7V+xgPHmpKkDiZmlzPhhmX|28a-n~1QR*0(z=Tcs_IRP>10y5pN?>@ZP&9+If81y|qxsU~HIoF0QZs^Ro{M4q`_htIfPq z`kb(#NwePtYf%np5FJ^OlI_ck9i-*crhx49S3~yLykEtP zM3~p_p+@@YnV;XDKrnYL+CN_|lq%%rOpnw?|55G1G=V1@kGJ6Ulo2PZ_QMXZId*2c z`M8I=2i5>B7@q(?p+V&f1LKEG=&gR}&V@iAoY z8RjH(*NFZv^1}qA=>a90e?v#hWDR~&S?^LEV)V7JYE~WWptSb8vL+P}zUG%DB+tuLew@T>& znFFvgBvfCUk%IJcb{ddaD!E5$bn!T=x+QPq#BqGwG)Y`cgGva?g?(4sW)^JNL2sm=pgNyc1y!Us@^q9*iuAM z3*uf>hh>U|Q{LMsbc%RWAz6s70#5icIy1-Zf|U$R^v^Fh7xfx(iTSUSw>vR zmBEj`Ib=Xiu>*E%zb0BRDw^2vX&w^IBM@xjPmQ@gc3Avu19>B4j9zO6%F*?twgFUQMjw`o3LB8P-d5-B zB0afLqfXm%{KA_k0m+M|vdc%P>m&budP3u7%fR~%S!^13Z)JB z$*PLbHOUtf${d`{Czjy9XIiGm1UBX?xP{{^MLZ>n%n4|jIq>QvC*b%RQS&h2<)1mB7&OgyD&-lq<6ePXA z$*N`&ZCm~zWIx^Lim+{gV{7|GdUqI)b z$r3t4CW>%wN&7&KiCC0YBWu+W9BGw3qZSgLO9UC1olXs>-J)M^rN$AkNBZlgZ|IOgm>BTGC1_)LJr%vOv`)93qwX`H4`=d*@1K^M4RNeTwx@%&NkE zt8i|XnKCmUtJk#f->yoagk-hsaT}*kqutpKtYrXWXW$c>#lZC77jk-U0#fMsc zDHaR@i<=z_0vnhEbStV?U!yv%4y2GpaEwy#oon8dfO{fGzj}LE-phr=b3Q)@+OO@Q z<&$s86O`UDeZ}>boG|AR48csqqS0xA+toJR&EQy;*y>`~%x*McGy_|&m>&0mj84^6 zs7<*ra6DZ7=zb_|J~%dYuPzxG2WO$*B5kh@z<6~P-eNm`rZ%nxw}?DG6ngRKvFFA` zcQ?CjIe~IP#WB7P9oI4hk~t3UszpWuvT3UiPPGTy?&kF+&Y2K zE8=dda^!MNm+uMLD(*pX*#LM7^gVJz6X!RVf0LZkNWcpI`K10yda*`OA;#&#heD2* zxoZ8YvMq5RChG*?6N_FP5B#i}uv8ZRw}pf}Mou?A8W8*a2K-v1T`-EP!;^jgr|waL7= zMbry-Ia87OpgA!eh>33prx;qE3V<>8pVRz9vD-xjH0%?URRCqa6(+JZ*`uoSR*WW)_{erQ~l6J$_iZ0i00|9dw zCtV~tAMf7t@zGss2B84fn^EaYJJ(roZzm{{*!GEl$-_u{E%8u6sm&po2P7FkG?7K( zszZ9sPYop%eRwu|LbJk6`I+maCFh@2P|t;i3{mRHn;1M)FeKpS7=CP*x@Bv6IQPTD z;OFlqJ{m~tH}8XS07uws9HXYp?amfyB=60E&O#*TR@0T_}sXvV>f z$hrqTD)c4E@2cWzWl2h#}rJgNzIWM+8G`DAkMm0VY z+!luT?&)Qeg@SErgr~|TQ*7Z>ce?b=oe74^`M1UA+XK&{w zE%(}iOOH%$iagwB?R{DLGMu;R$dWW{qp{wB=QLyX)pc{DrGZmyJBpMkMj`z?AaMCk z0;LI8hmTXrxz}3Tg|M%LTopb)KgOz%W`2?PUI3Us76EGL4)Z3we-*ZKxS44RR{{QP zVwi_0@d&O7x*;fDGJ59*Gs-$TOj8OX=7%aIJ<9Jr5-(+G;Ud3(+D+)#qcl-plV&I} z43w@7$qxrl+{KE8^g1*jzE`@|dP}db^5|JYl|7__>x(=~KLp|M?}i7mcWVyS%e+8_ zrs$ehtsVV6SLHa6bNbQe-z1~+Dy#j+KVJ~+W@GKFr-DMAJ)Nq9+e(l*MGnkJ`@6rX z?sM_awt9{di)6c?T2#MJ_Iu-1c4*k%7WJfRk3bTJOILWg14e+yZ_+jWtGv%GIQ2ih zuPT3H`qYrX>6JUam-YWf_<^LV&%l5X9!n6;jUu+&_~SaPD8@B}<_}FYshliBiEM7= zWSj^#vup4Mq!*k}71;~|CgWuQN??r$jQeg50udJbOLq7zy3ioyDPXS~5!_S%3M4Uw zI!TPew<#eouP{9J{VT*LJ+-U)UUauKm)`p9s3TJo=JzX*QQLX4#;azy z?wkH^S7tF*+Jo(&3S^{rjC{=R@{j;PX%7PavCfKg4b6>3;190Ti=wIlAfK?l&p8G* zO6Oer!Zb1xp9;ivo|t?(Y|gdmMMIuP9S7ZC3RQV5}C%etU2tf8)BZjfN*6pC!iPb?e&w@zSY_u^W zk&8@XO4;1XGOzT7K=&4ra;}uCjOtIwo}&N#Swi?r?p6Jsi<{1@b!g81S#8GqewZuW zcqOsnQbRWTg-_XX75|Pp|2{%L`l}-uX=+|@*2j0*5SzejKudCbl9MY0aiZcY$u>u{)BG1@Att!b`# z+BUC_g)H9q)WY*#c3WvbYzpQMQl1DramMi(oWI?6TA=#NIk~Ho5hdas7m_qXb%Hkb z8`oTH$Oi`c>QuawFMo60liNp8tqaRpYBOcX`)RZ*bQb1_5zJI4E%nOZD=bZDS?}C%{PVVC>dELspb)yim%O z)}o=vRhoLzzECF^T{J;4VpOeM%Yr>~QFMDOCRlvuh~bJLt!|fv;qT0xgf*t-yeN(^ zBdw`OvVd`=4x$uFS*KRFO>8xf3bvq1oW4W`1U?zNc8tp{%P8l5VpVlk|Ld2hq)i}C zycZ?Gn^y~(!g`=bqqM(_oz-GP(;MpSC=u<=-Wf&O<)C#4&+^^AjU%=S`;o2VdLXNR_uK;dNvo4B;hUc=o^tO3L>-uRvPWL z1xPpJsU3J$ek8%Oc$d=AQY5f>mhfu{sK*a|WVh_9+1z+smCP+)*v0a~McSKoi#Z|+ zY=X-Bu>9XTu>Y*{rPA^UwdT&Fr#gZS%( zzaRVYruouC4ie{%s5L~{r`EGoLZXn?w=FN-GT=CN%K-Y%gr z+;h{?eKD^=Dkm1u9wR0&JTOoAu?TIdKjC`ogwM&wqb?%E)}tdp#os%i#trcyzr`tRqz|s+DV&YqjA&{SN=EI_)nLPW&uRz(53uwV)=G zSm0*_r(;P#<#?AzfLQL%?Zy|j|CqjxRWJ#f>o_P}~A+>a+ zWjIwXSr;Kr%}dK3cw0^iN4^~#QTogjk8lQhrl*lefQF~BoVri z<9@R-9XIFK_M?i$qnPcNWWyozCrDRdxhYMRCS{G@ZlWr_v-iw7c$V2gu>aanVw}=! z$|27dZ2*;}J2Tqpn&tF6Gf20!_Tz}Xy}@s=zU+tFCTs-I z*cO;y3w0+WTvbyh`dDeE&gZVSc>CVIj*s1Be=+f9CGX&h6C)GjjuDJ7dO?_O#H~A_ zL`(yP@snb$1*?}?OI9a2*hs=ZQLU?eqp7IF`T}c9vQ!0y@%0zpBGcQ9tWn~4WOnz? zd6DV087;a8TQ;KtgL~_8jPa^d9@AUv*FBfsrtbJaqU4UFg`bo+FYye=yi&Eup_3yl z=od|+vnsF$(zX`!-i~B(vO-RR!cyzWOh0$&_8JWz?1`Jh(La?J?QytWF}7LQLugFr zzcy4NMr3!&CsJ5Q>>$ef)5P#V$-_tQoFv^(vJ2ibhQE5R^+h>UgFM)Uo>L?Yg0aTj zB`n32`mwqnU~6^<@wM+3JhL1xr>N8w{E95#&5ZMVS-eW?y1O$BiD1xz5Yq>6qEiJ` zD+=?Ag(5xwf)*x3>$H}k*<6nt!m>4bj5EjM>RLi_|r z-f4d+v*QWFV%oieFR;ysQc&P@AYHq`^pI!NOpY-DFO&eLTi7|#ZV>ra>1syLlmic| z#s=jz7bfw&aWo9SI)1Xn`o!Um0?nM8n>fU6mOYD&GFlMTn^H**s>JIF^aLNU=lK?h z6uY(Maf=w2KayotbbS#M+AY0K9%AwkUZCd2{Th2FL`VZ~%KfyK_LTA%f{bc?aYLFU zmF)(3X2TB;ry50=RD0EX9=x3MiS;_FU$F9ZMJ&p1->Y)RH5C|{z~WK77%zc1Ta+;1 zqWbXkT2V7K`WN41d5zVfo+uMb?>)k}N5gXGxLA_WY!=`nZ~O>vwQpaL|0eYTf9C+k;w6rU`4DptnK0K13?k=!%(o2Mztc56 zg9*oy|NNS^SDt2__o0lQNc`GxYA#^((aZvhWCfqY5{$aqQ0%CfC6$)O4C`bvT0{)j zvGBd({Hk)$0WcBzrARcwRWjw7*rhWhm5$#N7`!amtTyS*K*K&F@C{oKm-fIVyLnLg zc8mG?@()s9OP&F|xa`ZY=M1GCNt}D1-!Zy91Fd^}87uD0Xtke?xb4|@ zf+6s1gRWilZ*L}#BpJT!*JBayB0kOJ$nUONT926{&Hj+Se9M#KfYrCR?b%O^4enF} zM@kEZOKrVauXc@m;ar6{nUT%^g!2P>*l#v>SwEkq$Q&1>4)$HVuuhrd&Fe@?pDm6t zQg|l0bNuSy=1xpF0aGgQBN!ZF;m(tY5nHf8`+ZM^ke@PqL%VazlX9B<){UVf+d-9V zciDw5gsseMTaZIy$pU?mpp5saLayBbiU`5XFW@L9&}>7(U1oNjS=SBCc>XGvrf=H6 zbN4)M5_>xCwSV(q*nSCNk1l+)8_U(H5R#7{-i!Kcb_344B&qHD_G5Fp;ynrD8%Ymr z+4wId)3q6SNDqNI6y74CxeXeL@ zDdE6V5#|LIjzHpvHLa-CwIN|n>mT2_r5~7C&P#tx(8=C|u)4}doi~tKrJZChBE%Tg zbcebSr(UK=$9L4_`gDp(U3u$kX}1?cHgE;(o0!lI{p`IeFdA89a?x-Zq9LY)-1IW? z+31SjGp7Cz zQw!jZL<4}SBt;$cMb#i! ztoSiCjgdIeZr8tJ2Wfa}JY<#W1Z9-N2g>JsVXGhqW3Si#7}dG?XZ-x3(;wu4n&8DE zfh||l$&KdP3lsJeZbxUa*QEP|AO<02EMxJHL#=>+Y~tvg>VJ1CgT+C&P1^xk_O6d6 zV9YL~2|E-(SjfUI`_cH;Q3w8QMF3~V<)ykC2wy3@gScpG)#9W_`VZXo_#Mf zUEmnY{!aPL%sVL`YCb^_*rSJ)22S8%?%h&JsG$N3C|u-+CMgl#Jxc_|NXntszjQ% zF_deAJec^f^b@m614>1(h=E<`bF?WHR^$0ev@e6->!5Gl$J4WNZNgN^-OGq?<%02` zIR2JH#(Q+R28^m`MDzy8Nf@a6dOOUOyiYwSv{Nm|TWKiw5&ot_q@(p?I>n!Qpgh;R zp;tb`vLT?MCz0SkzCS?zd#V)MF?8PyYKZCvXezJS-Jt&hopvDIEArhcGYrGn^)@wX znEJbDHDNTN`(<;>EuE{x_UB4=%!$J^G}=1E4SzQmCD0-8WP9{u+a(HQLk zBbfnsNh+os|Etar7kjoKgK-w^#6V&0s?L>nLnWC|Y9pK)UODgi$tY31ZK$$fDZJl&IK`IS3J@sn5Nm>dLFKwcJGj(7M_#r!U>0w^WZUTvTOD9X+LA@P{h%R8 z!=`YCh4pt0Q*S4aWbTuN-VQ=qlWiw_gJk)>l_+L$YS8Bqqo{fe5lXtU42?x7(alIc zn{jwm?my?e99!N7YdmkQZ)h=6GiN$arZt=kz9<^UP5~d-3g`Y}x*&Pqjwu;Sxe7wS zM*xioq<`|`WP)K{V{o5<&=0CkH%xHX%+G1^b=KU-G*KbX_S)HQ!B8z5c^a@z68W!y zCh*S`2|X7Ag#8^a@rJzU1&c+U>w7!E-wwl$7}4jjNQ55|AvV4H6f?@;hw2;3ukr-( zO1V#Lqq|^pNJa2#=l_9RQ3NRb?j?pQc)A+gnc|LWYv?uP!G$ORYPkDG1v1tJWX?ON zWtSvU{=crNx)BelDR1ejq?kX;4rKO*D?XHb&Ulrvz8&e2LAIaQ(JOMpxs^fm;Yjf3 zurEIJo4lghhZjyVFH9PYA?8mZ3hFgMT2E8kEtExY8wW%=_G^CU3rX z;8gYyxb_OFJB`2A*}Jppne#<}!@`jc#IpS2pnn-iaMjEZQHbtRWG|Es+R66{nC&8g zRTu^QDKU`|evE>Mqva6>WX3>}LW;A)+KRo4nflTr)R~|1G3Q%Tvm`Uj|I)xSz@Deg zcp*E;IMi&8`MX%H#^jmyr>U>iypj*eFdIY?PGY1n4>8nJ!Tms-ZU&zWT(IH`-Y(c* z3D%pYZ8Jl|$+Yr93y3}+zL}pbHo3{!Bo0y7#<2`zqIkKci z-G6syosVEte`(A?RnsxQx^dNnR$JtwyvmBv0FU>)I|06d7U{sFSk?~Qb{&a*{KN|G z!%9v)a?WLXw*$aB4?~DrO`%fd@B*pZ5nNM6;J)A$QY>310;Ds&{q%p@d{1g@^_3Dr zbjEk2P@)-cBRFXnLoUw`!UToLO+(e6H*z;D#aD#Ba^IZAc~r@6R3o(4FRk;? z*gp_oRbkT#C+a^C`w5T@R65GWGR#D$DSuoK`_3|caItVSTo3)#u8D&U2nd3+WTv5n z1vE{9mHL8Kb@PW-KIGesO-MN)Ig|F^#YhHJ1tDbU7S){yk z>RLa2o&ys5>!Kcc!IHCtIRLV zwSLw2=j`W|)gRwEe^ZvY!E!T}ERVDu4@RU@)C9GR;`sjmE7Pgz&RT1W&3 zm`yKVr3R40y8-ThB)f?rY1h>8=5pN1cF|&+S9$Ps%5%uRn z2iOAn>b^fKbltkaFyrM;Khgk7`i+_O?pRISAEftX0H)X-qdwTIDIUHdj57l$=L-28Pief%GLNc6Pff8&S+Z3WCqqtwuN(hQNnK5o`TrE(QLswC z*`rJ|YupwiSwevMychljO9WS5(TQxq98lU~WMOJx|Kk;-3BcQ#A2EqJ0`tg7M5vMm zK`8`K&Z)q*wtUl9X?0)TKChdRT(yNPe<%|8^0nd_6V}_;O>vFk9g;&>auUM)*)P8UV_H*Unz>@QCC8>mI5)7)o4Ze2($=3~} zyCBBk+fH;JK$-0T9i_QuAfV1^zV8@k%1}W${?6j zj3NKqFFj>px(8Rwq$yfMi5jJB(&#Mlth2eOS)##Ni zN-RgGM#vAAe<0@Q=C4-ES8|&(J5$R7Kdw-^W^^s8{>)kQ7Ce|;a&aC# zEH}>)^6o>{?Yk0q|NA@BS1E^h_CO_IAI8zfo zjHc_XQ$WTk4geJ}b_mYAQMCtu4<+Whd29R}?H%?!-g-Ur#~1IY3T}DA>U&VF3epW2 z?J!q{br(W@0yJF7j#p>w!hBZFU#~suVj>%818w_b!iveDy|aBTti68(aohk_>-6E{ z3;0CgT(I2{wf`N>TKHQSWuTyYA~ajo7K;L>)nOzo9hy%Mh-drI5jG+ zSHbGX7vg764r+77cd#xjdP%$_%^M13TWv2GzaJ>7x8W$5x^674`}~6Ut|lH!KZ>IC zk@hll^=hzmtqxNsl^X-01$Da%@|FvOa;-xlefZmNAOS`_&>!xUauR3XC~pX+55pg5 zswZbO)iggpl*CtQtm313??av~_v4MrMv_uzvcg#4-;s1p>;^#kuQ^=@8;Bq>0WRQQQbFY_>oQbrg(=3-RtDWTV%8ytLLbts<8E}-r1(Kxn~op+$E*bU!H5w>HP ztz|H<(^x^t0M1;-Y#U*(8!i>0MD-ybHk+n95Ev4BCgaK~W%`kejE(D(At$iS!C&O6 z>g;(7i=*_snKCRifP0PEfg85<5lLMP^&8llYok1&&XfEG;avK{A$vE;qNx$?Z~r)& zXTFs<9k6Tz_2EA<+ifQ+mK=jqJX-6?hmm55vC+ZWa2*@pl5U zY0tZs`z952gIR*{*wklb|=l~e;|I&;R8nua?^^vKBF;+5`aFxnLwy?A)pj63?l~B%DK%h5xE4RI>JQDX7762r9Un=pxm>n$H#5W!$NpG-ON7V4 zS~!W#ku>`u+`6`|%AqJ-9e?$>k`B`hwNbNI%j`ZK{{L+|% znGI1|IJLw))`EQUMU__&S*)PAoy_tf@pDu}kG5%r_F8&@8oU_ZfKJxuWda7$Jt~Z5 zv#=&3LazMuX%aCQlDS|!9sHy|t}ohHmgpv}ZQYOfoOK@lErBk)Wcrq7k=?9?#TQ6v zeiO}TT6(j6-G9vh+JI>-7%o`da3IUboUFT?InZBGC^I<^SMvlYhWo8J#F_EAscH5eF>VtBD;gbZ_Mk4?(S$l4AUaWad zHfr|K!ud!1!a=(?pRPM&=^%S7yZ2zDvBJ%a077dbkUT2bjFWD^Fj|0CeDi>ZYgC8p zMCUJPa~S==mN(q3VG zb4PEE%`F)EW@;83z52v8&Hk>Ym^uq69viz+&A}UcfhFbrX?8AS_olDDlM0Fdv;6oO zxLE!J=U|r@VN@HMA7V5wc*CzaM5hlUBny|ghKDyS460}8VdsZC$HhlXULYe@b!Sq* z5%QHK=rX^>Y33-qZ;m2#TtQVQ> zny7|+e82~03s&|O$I#BhtgG&t+)@P0l4Ddd3cK7BW~_u$wQI)E?YQgn02hbqkYQo5#Z$%nvIgzbgd zq#k>)Ekp>CC-50jOiymE@ya%{eON(W(#Y<<`wVjP^{d(_&{lEJsli^Ghn`D?JbV=AR- zVm?$CC-DKTzj_usR{+&;aS+JmTX?~RkB{G<&gr$BDLSY3b-p^X3ZJVpVkG$l@M9^X zOzwW#g%B)mG^QP1O7|cYG#60HDtrCCPHy}D=*bzoI`~*B;}Sp4H}2xpO=cJ~Qej>+^juqSp*)#m zCZUA1N8#8Fx_@dDF)qI&Ytp(2VTq!4jhZ*_g0cZA3}p=w=+oD5@_+hNMgQ0CRr4{$ z*=5z%;74G`VIKf5>|Yr#5n6~*s9*%Vnr=nfA6M3JKDjhE2X8jgU%U6J-s%XezU}Ab zR!2aQJfyS}Oi^aEf%xtX6Y~acdZd^7($1ncTQL^~!zo$6@RVl*Qv;mxfB&*#jP3p# zaF3)=-7Mz7d;_RkqZ1Y63%d+`;Ik~~)x-bc9QG732^R%&UeZ)rkBdwb>MBr4JzMkh zg;k7bmSVE}m+S24Q(-t-uq?C`1}J`UG}RCbN&wgd$`FdM`2Lh(XF}>s^9=UZAe6qp zbH!cAM6z<_!nrfO^}?G_(gM6elp%};oTevsJ8RH?IbE|!BBz=X1&`cisl5zgD`Eo? z4=lP8+dyMlfg}ZsQUUuNU>$t@wByJzN75>jQ{jKS54Z^Y&5Ty$Z(#u@KLJb%VR#_? zy~SXC3j)sq9LO1d*JgY`KDg4P+K=JqH=<({kSDOx8j)%))DFTT{Q5`JA*Ep_yF&32 zoh;+>&W|~o){D;K_AxEIXJ$q)-ntyCmSutU(voYUV7mVKCvZ_af&VYI)JDmxNDW&dkN_r87gk z-?hOqjc&ZH6_*}gxMHvL(OaaMUJGt&uwCl4$TH81BCT9CQp)cQ5^?8nQC~x)z4u3{ zL*SU;8Si!7$fg5eM8}NwowIGM@eaQD^`J>`|GCT3g6YwwP+1X^W4JdIE$uGO7ZIla zn)!I4WdN<9@FG}%TBkA8oF+AN<_{;Xz8AdXU>5L$qjxRBH254v0N;D%VL6f_q8Ffh;gya zW3Kg>OUX<(T~N)j_c59R;Ya>_N{dhO6K-DoIXPA_ea|Axv(Vs1YLu~WXpz7-9n(t3 zpv0T~9Hmuu{)0!($q6%Xzt=(5Opy}UzbDF!Qrf@W3X=RS=8u+I`AU4g$jQ@|$i6u& z^*g%g?_czv-kbljMtpwmyJ%(FhY4E(tuG8XDHz&`bc_B`ektU{w%r&)F0I**+sl8g zniAh$|K_qN^;_n-o9@z2l=fq$Ft}H+mfByH70z?NTh{m{cZ{RF&s7{&uE7z-H-I*M zrHagwli8V(Y5%0-@#^>9Y*(8sE0Q}olFTi6IiNScJ4ArkM3>E(U7eQGRq^?arN9XCsNT7e>gJs#0+eUZk5LPd0!DK^yaEJ{si1 zp7yD?O1#E4zUqXcQm=AOVsC1OA$%rzw}fpx3icdD4FZbwpiI5YkNyV|E#U_sgqafh z2?m&MdHThoj?(4GrUbiZ8hp^6LZN98qimUf2whHd_H*V97P@7s&g~zbi$+!r@;v54 z+E8V(mtjptKcS99`ZlZ4Ige zO$=OCJ&fmQEVQXL8X-?zHE3%7zU!S-xM|Sdlwswv6 zKVhAJW@t{!s=XNrd0Type%8};)v`aLSzl!KHY&3|5Cq5X_z&p$MgP@my*JN-zDl!lIV5p0r4&*`NPr_DA zzJsx=698ET);lX{mYn&TJZJI6LP}6)mdoiGNkGZ*7V@{5fs7qV2S3`o zc$5&IQuTXDbbLegZkFHK@bFV7vNbQ`h))fA8RKd$4&XW?DH~4dzbkxD= z_015>lS2=GKO5t>I`&=1!56lh$9gS zqx7Up|8*a(Xm(?tJsB7?!KLwyO~zZln|p336{FjEEzDv47G_tS^ff+T)BD%3&4^Y^1J8MzV3W=n73$EHhE$q|IXYjl1zdzO+Ev$6>u(g&+JR=F!g+Prm!6E^GN<^sQQHOu$5i#*kQ}j-pu5%3)8d=f_4BCF__ROp34ex`Q@A_ITE$t*k zy=SP91^aYBY3Ap*nNvWU*n(XKHbDO$w+z%7)$@BB{*cClj20u zjEPVrL|XarP>~j6rIC8B4;z}|7hXDu^4#=l=kF~-f55l$N$@kL`XA3fFbN3fBJ@T|F%$l^4Ei|o zjNBelb>$RG_%eN0ePxY`S~c55j=EZ8loODzi#TL&yB2{#p5x2OEC2B?=>9J8&%&l_ z__G1{XpgdHc=?Yi#5Z%hCzMZz0@f~?74FTp&p%tvaljOg+p0oc8 DQr=M~ literal 0 HcmV?d00001 diff --git a/docs/pics/planet_github.png b/docs/pics/planet_github.png new file mode 100644 index 0000000000000000000000000000000000000000..67efe9684c29f66878f45514e84a7e61e66b1284 GIT binary patch literal 8309 zcmZ{IRa6vg)b$KCq;z+8m%z|SNFzwMlnfvsG9%rfA`S>BDbga{HFS4(Hw-Xz3H-eO z#kanj@8UdXuXEOV&N^|kcZ{Bn8UZd1E&u=^&`?(~c=82L_6Hm7>8)8y*aZOaU0sxw z^)!@~ne@Co99f>lIWp(}_ zGDJG)w^e`Udi~x=Wvm)dQ4l?AO6F(HSHN{(r>JkZ9FFmR$r#~_@HqN+HTd)KyNvfS zJwVK_omYJ^ghDquKqKsVnBc~i;vzNx0q2BoqbnHh^5<=9l0gMJ8w{wJI$g=0YsUEU zzKbN0#GhsBEN4dG2h$`DBsv0rc@pmb#BAd0%|4;Fa}u^@!&30Z=mdh9lL^aNE{S2V z1oAq3J9B5j5UOtUCQ-!3;4N)XH<5gDs(5%Jj3C-9TAibBOXpEu^Q?tk&{%G_gGSV+>u_g2v4TSBFuD-{v9@jzl0}tA$($14wXjIQaW3B#eLr^OnL65O5V4Y zw$Gr$bXYXI9#y$k3$BG@m28znI?1B>@l-U%h+GwrYy&ZIhI3zd$SSfyZaJsVU;IlV z>cQz;YkL{^m+2J^MSg|ga00>hXE*Xs&&PQDg{*%NW&I_b9^81=N$A}sfYMp39Kc;S@;=G&uX~>O z51Pj&FQG`yY4u}>p1$Y{$Qt+?KpPI=_|}mGy*>PM80y;(7iz^c_=btms-H`p z1_ukVM9x6JTMIyr1dlVOx zAascQsyhJd?AlQm`qG&d%woY=Me43gGL-y4v_&=g?54x8y7wAEQ7686-Wui^)cfqG zCzd-lou}_^R0K$Agh6lMbysl#?iuK)#R0W6p*IIl@8g0|Nkv2MHna%*2;!J2UKF>K z#)(XS$@*9sr5boX@=$W?KdbiO9K+xJ%+{$qVHWlmScIvC#CRQ5ZPP;rb@q{^Hn9v_ zUdLMR3jzNQx~BE7`1G89Prz((uN8N_)qW$`z=~8uB@z}v?*_Jf;$}F-PHz7%vpq#;#_CQ)y8us zQ*imztt$n4;Aq!P;0NQB0cwO8jaPC0DG*^}W-H>W1GsHizF1+;)RSppinf9EFkZIe zmQg?e=B*#fXp!h3MSf=hXB+i9VhgO9Hp{w?(yf}K1j}J^7>erH1X-+D(cE?s&zVZ1 zB}szAS@hNE;zM&-O{1yctfFvCb({*^xnPJg58N7_-K(()9**bpN6_Wl^gs;E9%}~^G1BCwzr&BON*ZHx zt3%gD|CpC!gV;y0RWU%Jf+5OOYP~F4>}13*ab$4Ru=Cff_L-VENn-GS6O7>8&`aiOX+S=Lug#!hR1+kM0leJsONn<=N`ObvS%{1=`I0?RDS=)bAtyhihCXY2vd85s%owThoq%>qOq@0p0 z)>zeRHf;vsFPVlEdlhG!NmhTX+M5@c7oLaBYgS)5XU+tcD36y<<4mm`@J-H@c)#2c zrq4DXoqNv^01N=#i#01J7r15*j15imk1gdd{8=c@&)z9X&XLqEwK)D^C4+8nBKrJ` zMYmVCVmCo|zHN7hNe4zp5`GcG2ZveE{o1YKE!w@^{r&CCg*>$off9ii{%e{^nnlS@ zfc(;n~%{q;I?j^BS!#54x=iz&Jc?=0%gAPf;sXvz-RUL^Py(--tiyZ^DCAMR> z&xWp2S89KmK+WHn{VjK{*sT$(rn8rtsOUfaNnJd$d&fs&B2T5>gf@; zbGPRb5fVlcR}hz)keXKzUP$R2@nO@U7gA!*vPSJt78!B5b(u`32hs(>J2;|hh{K8H zSUXvHR2SdpNLSF|(G}4#N(-m`PTQ85l5VDPkrMIQ?1ww41^QgR8}?aV^PhRq<4`r= zpZqadvgEP^JmWRv>Mm9*R4e7~<6gQ%zleF1y5s>HmF!c| zyqe$jnJ}M`nf5EZl41N8Q$N$v(Dbm}HnmXsvVghD*;r;1 zb*Q?hx@&=sv)yjwD9Co(*3{O>c4>`aO|+es6`L4N%**QT^+gJ-!=CR~HM47)oyuAW zKY`P-Fus^m)5zZM`DL#fXDYjGRb^E5yg>r1^PS!6p2fK8m~@i61(WsJ+43#>%$OK5nL_>O12SNiC6n{hs}Nac=M z%NdqXE}Wg(vt7K=ya_s)LCIh~gc^l9{e1XYB$DZL<5WDnPlko78DYs%;~ci@^Fie@ z2}vj=EY;|B>z&|zwNtp2yF*WY5bfPLl%UO_o4Uy1`Hf6tKL1+BR3_J)hI@%y`3>nC zpPL3$KH`Nt@1w>;8Pl&{O}~8Kj!!C2ROQFyS8LT~BHr9qwwJYAJFbdCMN=G8hSDU@ z*7&CXx>dsGs4G*qO$JS$=i9qqFD{*Noq28F%dNm3{@ff>un87eJ2$;MYVVPccr-N1 z(P!gtsPFX{OX|9|8(W`dch{U2yfGUTMCh$s7?_>HxIOkFGA#oI?&^a~w=1T%G6p0| znM#OC>&ivmZ5zMf=CY#?6NcT(MG+b5`R5V;99T@+8 zY*MUS`v`k?^AM{_n+*9^7SpoVzx``y%KS_1cgW4tZZD>5P;TPn)8x4eF$;J#@=Nfl z4W!_5i{X*jqG`q1fWS$~<&tM(DsM9n?-kXdsa3@5z9Q+Xyu;=u!8g{H)%F)P_h0VW zL9M`>5XzRk$B!E}n{9c@%Ta#_&FPn+;ek1IScN~WCM! zlMmh-ybH!rkGQu*xqcSDJ+6BP50aWIG)+qv!}pf2bRi>QSFqzdIkGF7^P`8|`x9jT zrtZjXULZ$+z@6Py?t|;*jNd)%{+^`dy9|mqxGmDr5e?-?38*3i?4lM`sY}6l&#-@k zTb?nYGRsbzo-<_t3_EmAXh+;JRsoP{Z)QozC+q8Qrr?Hp0`bINktK^n3qvRg%lYM1 zi#S%WAz>?}OHj~C-ot~Sj7?cB&^hflIE*J8dj#M7o_x1KE=b^7@%zrivDd$H&?oH0 zwz&Hiz$e+bCHIv-giq|_xr2#@qqa7H=Lus2fKe_0j3)$qGMXm?0MLG+{ZFLe7yAF; zKmR4U!XaM(0MZZ*6-7fR@X*5b%bOk}#K_U!-VCzogVRp=vN5FRLz8WKmkP$Te(-8< z2$u-P3CX8xpi(#&u{zTR;FDe;HLnu3-WP0nwAbIm*(mx|IcPaeOA{TMV-5p44+4gc zhLKH-_Xt5mgRDhUKG@9O2 z;OU!a39+hq_ERP8i{eNf5*TI^n?lWgBpPBWT7#uDI2oh&^^#s|Ese0oCU?=!(kCcW zP|+a;b1FG5b}jfStef7b8oQt`sUKOrPuTMH;i`&;`qJJ>@+uOfzf)j_#R zBZv&8jrCGOSh#4XZ=XIINNuR@4g+RIx)U+M**M^h3+6`DEzKuLYB${LjF7i}1V*hM z9$-J!R5XOsuX!uw3x@ldA{@f?K6)HdIH#}vmUz`zgz-W(KhH)ZManSEQ%ey4#@CH# z2-mE=zqYv7!`-#6)ldEYmsB;C)pdw?UrUOnln;}k^(Ch1^WAAgiKM$TyW?uJoCm#9Air8@Kv$3#ChF0$291eG3CI{gcrqTq?QhUG1B;abBGD!_s5#+*NI9&Pkpr zzCS%x`PF8Uy2K(Kfg1y!teIzzH>Pf(=8CW7U1bMBMX$(EsV`JdKVUX(0z9@e&;l!a z)t7ZlWk+al04jsuT8uoHUp^gD>jAUnf6kSO-+V@aOyU)L7_J+~Db?>-fmnjiWPiSR z&~dpO^Q`7Z)Rf!fUU#3sFH6v@%$#XuRQoLL_iioQ6PP@;q5R(8v1l zh$2)Nd^ISof}K~3ln9D(QOe&ameoa6*aROAE!gI_BGeu>KbMV)?I$}@T~UX)r*Kgv zWj)S2A?$t@R1Am^EQo2ns^w?3&2r=*y>#9erj2+*!e78aGM#LlR<5U$5`Z%~Xf1ym zp041rR8{4KIC+z-rtF*nZWYz~5)3S*m2O1SoB@{FI$uys%-1|t>+PYTxx-OcAZ_@s- zq7;nQNWbhr2uk>D08SGGgIQ=}^GL5lwo~N}+8NN3hd|ANedu8HABv1jTf%$x^PpRN zuN*1#Ajv@@j8u#u^9O#}Z_)0a3QA1AJ4m1qRYymJn1Tez zIh4=w{5I#S)bbe$uqRG15nL!mOFd#m*Gu7 zFR3n@gZePQ3buW3nPbHH))Z*_1~}YWV;+UmqbTb#GAfGS7DH?_M@WK{v3!ACevYWV zB%B=Chkd^f4&wnIIUE*Eyru^j%o#pD#^XS1@Mi=kw1R)zsu@S61f=u^ev{Qg$9Ig< zp2hkQY@3bn<2r^7PdW!W!iRuHGsn8@mNNm@=`!#$(t3g%weGT?_IQt#r$F7hR&!}8 z;_zEb1Rn8fA~xo=Kzq`=p=WdS_8-)Zn#ww|gL4`2q~gB^%tykj-cu3+9Q{-e&}vA9 ziY>Zn`W=0r<$NG;v{(Xruich5IZbwcrY#tA-$8l&A?=CKIYBFaON6EPTqQ>_(zNIZ zpGx|Uuy6R680oLw2B$e7$A?XEjsU#UH~clRVfYN8u_NEwtTT8s>cwgePQ2!F$T8v= zoAHVwusDdiG#Wy|uVgEz=CD=THr)P+ceMEY5I&5uNN*mQD4S+oqQ0C>wSA2G$yoMV zt=Pi*w9wC&m4k#8ae_U74i@>Ljba|Z7DL@84c)9g%V=)5y+Uy$i;G-SZ`r?kQUI!3 z14i0n?yel&_C)+dqc(vI(RR(<=fhM@IL{p8p5E@X(%H6{h&yy?Pb>ce~afb zzJyk0T*$#UM>GTMd<$1gR}6!;b}M|U1~rJ*l^KI9Fc@vQ$Fb4YSC76bDzPElBVZlAG-HZ{X>(IG3D_gmV4;U0apuS_M zoK1)C-t0R4-F_wGz@_rv3U#0 zMmmSFCh*qgU1=YFd}Mq$M4F+7l0BL|i=ZQ&tzG7rqjy^d1V3hM&;8v#U>X%$oQ+;x z*GrWJLGFa_vjNl`yME2FFz05lp~Syai9ut;f~e%-It7a#->)h) z@^N(8pQ>T`VK(wrjX%I_OAgHLVuTKYi2<56`uFE^%DhH!k3+UHdZd z^~6&_kIL{aecallvO_NyawI-9$@Rf*u0^ai`rf*&EsICqf28L(&@5ONU>!HMiE8cq zC7v;nz_&5SpLE?EB&@qx7YJ_la{u6vYv%Wm=zcMo}*(f?k&AyrP`*nDxtS(e#`uCHf(5vsjk1!EA6c2Zq z5Tp(*4KAYURT$)VTlsY(JYV#ET23L<+1Kj1jIn2VRL52%Nc`J=uUFCvB#I%Urbr&h zw$Oe3+C!MXIVPQ#|sRxYI(Yz-nJNvh8BaOXO5M7fp|o6#_2erG*+V+ z&sAgi{a!pS5i-c#wh9GmhwRVjTK1UIs7{|0D6|V1*V7RdgFl+`bOrwg z950Lnt(IHM#SdPmWE|@DpU7D=tm;}ciH~L?D~k)qT1}?()ySFpyZ@ zhgByF4jcOK++Wj$U3hXWhKaiJBD{Lx1$&WR!)xp ztu_i$wLF$vkNf8fy3D*4xV*Ufy5*UIuqf0aaOAURb?z*6{KA%m*9f^If-B*I%l}6G z+1jF&2b=cqfc|8-&P%KcZlK5jk|+DaZN?R)=>c0PR?u*|9Bdy+WHq~w~Z62v|JRzg% zy;+ApFw900kY~!DG;N#|wojv;ehdq|{>-^@RbtV6JLoMadqbJos?z!aCKpIWx{55` za@AN;zv~LDjhrbr&U50x0vT^_UmXw9(J>a;O_vQ>|EupXtM89zJU;n6%X#^aUAH^c zRg&T}RDPxQb-^ZFu>Qk$>4cY&?&MD z&#yb*;*~jX=^+FC%`s_ySnI%i+vaSqjX;2B&BuSaXC^c#5kaAN*RHo8)JD)(hI!4` z#YXg4uoT%AN^qlsTgP3e2e&jKYtu__9Cst^d;CGxqc#t!48}`;@?p}?9R9U!Fa{3& z;7OjYLWDg0JNjiXFP@@%6-N5Qo43oL(7-b+EWtixC#|J^@Ixq4(z|VPcJ*S8z3Ha9 zeP!JL;4BEkI@(C7GzQEVtK-ZRrmEbVa57VxVa+{#OJ8YN=SVw6=>P8VCbeDU}Fva|;;wuD1y4I$^y1?^m8*qDA1GDBTh+KHs=pAWqpw1D3}`KbQjx+0LR9htE z-u)^D&9Ys1At^N2`4FQaXX;n-f?-v*d1bUHfZNrxeAjPEwA8|Dc4XDHS6wC$Qp9>=z;#qjL7?~_rO^PwTDafx^d~yP~hW*q+=9TLRb8U zb4j_VApNbu+J|`wek2~=*1oR*!|21gq~9|ayE1U=v6M)?nz8_*=*YI}2XR#P{^4iq zw@MwgNYL*_KZg2y%3j7Gwgz@J#f-x=I_RRXpmX<1251mp&L887=(HfBw za2d;%(vGS$1iMP#Z3-*P|4j9X8lekPH*l>wD3lX0lA+u%v?B#@q11Yi4$#XZ=xys6 zrd8flNNq_-?Voajc4AmWyMRjY%;+rJBD-a@l2x#AuP?JmDo(){&X7-@h(0T?y2un?O0=JP1K1#K4FwTm;=r?3KB zsfZlzizZy$Yv>w1tSC56iMTWrP0k9cw(Q=)y`VlLK@lz%^`$F;CJSr)1(&WWB>SZ* zL>0aK(~JT}xHKyP&qhRf{3`{ZAZLkEDH*DrsE_+sa5AZJmN72WHhPncsHLJAJy6Y1 zl;*QXjQbQ`_)t6#v1+0*7qXp~nP0_;7FUDv_c#0~9=ICbUOOJ@<^B^lmi+u>^AH!> z2V5SM#r)?8hc{UQ1*KY3>(%<`1=V)Z1xHmi&**YWnv7t}Y*|>#+P0J;FKKD61ypvX zi&6=)L;?@W4>bs$IC;3+fhzB6Ekj4fOb zKDz`;`9BSb1(tZ?FJwRVJ2(z=uqgo-D#uaotl!#$=D6G^a9<07DH{!3swz1AyX>xX ks^VVt!}@b5ejWr&sK+u{=a$9!Pn6Smp`%iwWE1-T0BSZF_5c6? literal 0 HcmV?d00001 diff --git a/docs/requirements.readthedocs.txt b/docs/requirements.readthedocs.txt new file mode 100644 index 0000000..1094240 --- /dev/null +++ b/docs/requirements.readthedocs.txt @@ -0,0 +1,2 @@ +tensorflow==2.6.2 +recommonmark==0.7.1 \ No newline at end of file diff --git a/docs/source/Examples.md b/docs/source/Examples.md index 2e652e2..e7efaad 100644 --- a/docs/source/Examples.md +++ b/docs/source/Examples.md @@ -39,13 +39,19 @@ if __name__ == "__main__": # 1.Label Encoding for sparse features,and process sequence features with `gen_date_set` and `gen_model_input` - features = ['user_id', 'movie_id', 'gender', 'age', 'occupation', 'zip'] + features = ['user_id', 'gender', 'age', 'occupation', 'zip'] feature_max_idx = {} for feature in features: lbe = LabelEncoder() data[feature] = lbe.fit_transform(data[feature]) + 1 feature_max_idx[feature] = data[feature].max() + 1 + id_count = data['movie_id'].value_counts() + mapdict = {t[0]: i for i, t in + enumerate(sorted([(k, v) for k, v in id_count.to_dict().items()], key=lambda x: x[1], reverse=True))} + data['movie_id'] = data['movie_id'].map(mapdict) + feature_max_idx['movie_id'] = data['movie_id'].max() + 1 + user_profile = data[["user_id", "gender", "age", "occupation", "zip"]].drop_duplicates('user_id') item_profile = data[["movie_id"]].drop_duplicates('movie_id') @@ -76,10 +82,15 @@ if __name__ == "__main__": # 3.Define Model and train - K.set_learning_phase(True) + import tensorflow as tf + + if tf.__version__ >= '2.0.0': + tf.compat.v1.disable_eager_execution() + else: + K.set_learning_phase(True) model = YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, user_dnn_hidden_units=(64, embedding_dim)) - # model = MIND(user_feature_columns,item_feature_columns,dynamic_k=False,p=1,k_max=2,num_sampled=5,user_dnn_hidden_units=(64,embedding_dim),init_std=0.001) + # model = MIND(user_feature_columns,item_feature_columns,dynamic_k=True,k_max=2,num_sampled=5,user_dnn_hidden_units=(64,embedding_dim),init_std=0.001) model.compile(optimizer="adam", loss=sampledsoftmaxloss) # "binary_crossentropy") @@ -165,13 +176,19 @@ if __name__ == "__main__": # 1.Label Encoding for sparse features,and process sequence features with `gen_date_set` and `gen_model_input` - features = ['user_id', 'movie_id', 'gender', 'age', 'occupation', 'zip', 'genres'] + features = ['user_id', 'gender', 'age', 'occupation', 'zip'] feature_max_idx = {} for feature in features: lbe = LabelEncoder() data[feature] = lbe.fit_transform(data[feature]) + 1 feature_max_idx[feature] = data[feature].max() + 1 + id_count = data['movie_id'].value_counts() + mapdict = {t[0]: i for i, t in + enumerate(sorted([(k, v) for k, v in id_count.to_dict().items()], key=lambda x: x[1], reverse=True))} + data['movie_id'] = data['movie_id'].map(mapdict) + feature_max_idx['movie_id'] = data['movie_id'].max() + 1 + user_profile = data[["user_id", "gender", "age", "occupation", "zip", "genres"]].drop_duplicates('user_id') item_profile = data[["movie_id"]].drop_duplicates('movie_id') @@ -210,12 +227,12 @@ if __name__ == "__main__": item_feature_columns = [SparseFeat('movie_id', feature_max_idx['movie_id'], embedding_dim)] - K.set_learning_phase(True) - import tensorflow as tf if tf.__version__ >= '2.0.0': tf.compat.v1.disable_eager_execution() + else: + K.set_learning_phase(True) # units must be equal to item embedding dim! model = SDM(user_feature_columns, item_feature_columns, history_feature_list=['movie_id', 'genres'], diff --git a/docs/source/History.md b/docs/source/History.md index d0c9f61..3fbc964 100644 --- a/docs/source/History.md +++ b/docs/source/History.md @@ -1,4 +1,5 @@ # History +- 06/17/2022 : [v0.2.1](https://github.com/shenweichen/DeepMatch/releases/tag/v0.2.1) released.Fix some bugs. - 10/12/2020 : [v0.2.0](https://github.com/shenweichen/DeepMatch/releases/tag/v0.2.0) released.Support different initializers for different embedding weights and loading pretrained embeddings. - 05/17/2020 : [v0.1.3](https://github.com/shenweichen/DeepMatch/releases/tag/v0.1.3) released.Add `SDM` model . - 04/10/2020 : [v0.1.2](https://github.com/shenweichen/DeepMatch/releases/tag/v0.1.2) released.Support [saving and loading model](./FAQ.html#save-or-load-weights-models). diff --git a/docs/source/Quick-Start.md b/docs/source/Quick-Start.md index 3053598..271ee44 100644 --- a/docs/source/Quick-Start.md +++ b/docs/source/Quick-Start.md @@ -1,7 +1,7 @@ # Quick-Start ## Installation Guide -Now `deepmatch` is available for python `2.7 `and `3.5, 3.6, 3.7`. +Now `deepmatch` is available for python `2.7 `and `3.5, 3.6, 3.7, 3.8`. `deepmatch` depends on tensorflow, you can specify to install the cpu version or gpu version through `pip`. ### CPU version diff --git a/docs/source/conf.py b/docs/source/conf.py index 4e882a8..fdddcde 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ # The short X.Y version version = '' # The full version, including alpha/beta/rc tags -release = '0.2.0' +release = '0.2.1' # -- General configuration --------------------------------------------------- diff --git a/docs/source/deepmatch.layers.interaction.rst b/docs/source/deepmatch.layers.interaction.rst new file mode 100644 index 0000000..5492bef --- /dev/null +++ b/docs/source/deepmatch.layers.interaction.rst @@ -0,0 +1,7 @@ +deepmatch.layers.interaction module +=================================== + +.. automodule:: deepmatch.layers.interaction + :members: + :no-undoc-members: + :no-show-inheritance: diff --git a/docs/source/deepmatch.layers.sequence.rst b/docs/source/deepmatch.layers.sequence.rst new file mode 100644 index 0000000..6e88433 --- /dev/null +++ b/docs/source/deepmatch.layers.sequence.rst @@ -0,0 +1,7 @@ +deepmatch.layers.sequence module +================================ + +.. automodule:: deepmatch.layers.sequence + :members: + :no-undoc-members: + :no-show-inheritance: diff --git a/docs/source/index.rst b/docs/source/index.rst index f5b25cc..7ebbb81 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -18,12 +18,12 @@ You can read the latest code at https://github.com/shenweichen/DeepMatch News ----- +06/17/2022 : Fix some bugs. `Changelog `_ + 10/12/2020 : Support different initializers for different embedding weights and loading pretrained embeddings. `Changelog `_ 05/17/2020 : Add ``SDM`` model. `Changelog `_ -04/10/2020 : Support `saving and loading model <./FAQ.html#save-or-load-weights-models>`_ . `Changelog `_ - DisscussionGroup ----------------------- diff --git a/examples/colab_MovieLen1M_YoutubeDNN.ipynb b/examples/colab_MovieLen1M_YoutubeDNN.ipynb index a5bd90e..8587653 100644 --- a/examples/colab_MovieLen1M_YoutubeDNN.ipynb +++ b/examples/colab_MovieLen1M_YoutubeDNN.ipynb @@ -3,17 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "view-in-github" - }, - "source": [ - "\"Open" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", "id": "rtox72csOQUN" }, "source": [ @@ -25,7 +14,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "bTWHz-heMkyw" }, "source": [ @@ -34,52 +22,46 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": 1, "metadata": { "colab": { - "base_uri": "https://localhost:8080/", - "height": 510 + "base_uri": "https://localhost:8080/" }, - "colab_type": "code", "id": "yTl6d6jO1oqf", - "outputId": "19f1902e-17a6-4a07-ab42-f012bd286eda" + "outputId": "6f4516af-0f03-4f35-8f0d-80e803021095" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "--2020-04-10 09:06:19-- http://files.grouplens.org/datasets/movielens/ml-1m.zip\n", + "--2022-06-17 04:30:51-- http://files.grouplens.org/datasets/movielens/ml-1m.zip\n", "Resolving files.grouplens.org (files.grouplens.org)... 128.101.65.152\n", "Connecting to files.grouplens.org (files.grouplens.org)|128.101.65.152|:80... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 5917549 (5.6M) [application/zip]\n", "Saving to: ‘./ml-1m.zip’\n", "\n", - "\r", - "./ml-1m.zip 0%[ ] 0 --.-KB/s \r", - "./ml-1m.zip 7%[> ] 443.20K 1.85MB/s \r", - "./ml-1m.zip 100%[===================>] 5.64M 14.4MB/s in 0.4s \n", + "./ml-1m.zip 100%[===================>] 5.64M 3.47MB/s in 1.6s \n", "\n", - "2020-04-10 09:06:19 (14.4 MB/s) - ‘./ml-1m.zip’ saved [5917549/5917549]\n", + "2022-06-17 04:30:54 (3.47 MB/s) - ‘./ml-1m.zip’ saved [5917549/5917549]\n", "\n", - "--2020-04-10 09:06:21-- https://raw.githubusercontent.com/shenweichen/DeepMatch/master/examples/preprocess.py\n", - "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n", - "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n", + "--2022-06-17 04:30:54-- https://raw.githubusercontent.com/shenweichen/DeepMatch/master/examples/preprocess.py\n", + "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n", + "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", - "Length: 2049 (2.0K) [text/plain]\n", + "Length: 5642 (5.5K) [text/plain]\n", "Saving to: ‘preprocess.py’\n", "\n", - "preprocess.py 100%[===================>] 2.00K --.-KB/s in 0s \n", + "preprocess.py 100%[===================>] 5.51K --.-KB/s in 0s \n", "\n", - "2020-04-10 09:06:21 (34.8 MB/s) - ‘preprocess.py’ saved [2049/2049]\n", + "2022-06-17 04:30:54 (71.1 MB/s) - ‘preprocess.py’ saved [5642/5642]\n", "\n", "Archive: ml-1m.zip\n", " inflating: ml-1m/movies.dat \n", " inflating: ml-1m/ratings.dat \n", " inflating: ml-1m/README \n", " inflating: ml-1m/users.dat \n", - "TensorFlow 1.x selected.\n", "\u001b[33mWARNING: Skipping tensorflow as it is not installed.\u001b[0m\n" ] } @@ -88,16 +70,14 @@ "! wget http://files.grouplens.org/datasets/movielens/ml-1m.zip -O ./ml-1m.zip \n", "! wget https://raw.githubusercontent.com/shenweichen/DeepMatch/master/examples/preprocess.py -O preprocess.py\n", "! unzip -o ml-1m.zip \n", - "%tensorflow_version 1.x\n", "! pip uninstall -y -q tensorflow\n", - "! pip install -q tensorflow-gpu==1.14.0\n", + "! pip install -q tensorflow-gpu==2.5.0\n", "! pip install -q deepmatch" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "p9UxNHuPMuW2" }, "source": [ @@ -106,54 +86,22 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": 2, "metadata": { "colab": { - "base_uri": "https://localhost:8080/", - "height": 496 + "base_uri": "https://localhost:8080/" }, - "colab_type": "code", "id": "C_ZR6gzp1E2N", - "outputId": "8401132a-5090-464a-879d-a27492416f4c" + "outputId": "903724ad-114b-4ea8-d0ce-151f9f6d4cdc" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n", - "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n", - "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n", - "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n", - "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n", - "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n", - "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n", - "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n", - "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n", - "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n", - "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n", - "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/initializers.py:143: calling RandomNormal.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Call initializer instance with the dtype argument instead of passing it to the constructor\n" + "WARNING:root:\n", + "DeepCTR version 0.9.1 detected. Your version is 0.8.2.\n", + "Use `pip install -U deepctr` to upgrade.Changelog: https://github.com/shenweichen/DeepCTR/releases/tag/v0.9.1\n" ] } ], @@ -172,7 +120,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "fQq6O9XAMzPF" }, "source": [ @@ -181,27 +128,21 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": 3, "metadata": { "colab": { - "base_uri": "https://localhost:8080/", - "height": 139 + "base_uri": "https://localhost:8080/" }, - "colab_type": "code", "id": "lcO29zFb21Od", - "outputId": "97d9023e-cbd5-43dd-d2ef-769e3a34cf2c" + "outputId": "ea095585-f5ec-4d1c-9ffa-117531a5ed3b" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:4: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n", - " after removing the cwd from sys.path.\n", - "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:6: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n", - " \n", - "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:8: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n", - " \n" + "/usr/local/lib/python3.7/dist-packages/pandas/util/_decorators.py:311: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n", + " return func(*args, **kwargs)\n" ] } ], @@ -213,7 +154,7 @@ "rnames = ['user_id','movie_id','rating','timestamp']\n", "ratings = pd.read_csv(data_path+'ml-1m/ratings.dat',sep='::',header=None,names=rnames)\n", "mnames = ['movie_id','title','genres']\n", - "movies = pd.read_csv(data_path+'ml-1m/movies.dat',sep='::',header=None,names=mnames)\n", + "movies = pd.read_csv(data_path+'ml-1m/movies.dat',sep='::',header=None,names=mnames,encoding=\"unicode_escape\")\n", "\n", "data = pd.merge(pd.merge(ratings,movies),user)#.iloc[:10000]\n" ] @@ -221,7 +162,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "L0yCWxQxM3se" }, "source": [ @@ -230,107 +170,96 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": 4, "metadata": { "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000 + "base_uri": "https://localhost:8080/" }, - "colab_type": "code", "id": "BMOvk_de2ML3", - "outputId": "1e43a5e7-f71c-45a4-bab4-e1d6b1a73669" + "outputId": "24448edc-9100-4a01-c13f-c48d0a6632e0" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 6040/6040 [00:12<00:00, 489.02it/s]\n" + "100%|██████████| 6040/6040 [00:17<00:00, 336.80it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6 6\n", - "WARNING:tensorflow:Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AttributeError: module 'gast' has no attribute 'Num'\n", - "WARNING: Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AttributeError: module 'gast' has no attribute 'Num'\n", - "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/deepctr/layers/utils.py:167: calling reduce_sum_v1 (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "keep_dims is deprecated, use keepdims instead\n", - "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/deepctr/layers/utils.py:193: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Deprecated in favor of operator or tf.math.divide.\n", - "WARNING:tensorflow:Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", - "WARNING: Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", - "WARNING:tensorflow:Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", - "WARNING: Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", - "WARNING:tensorflow:Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", - "WARNING: Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", - "WARNING:tensorflow:Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", - "WARNING: Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", - "WARNING:tensorflow:Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", - "WARNING: Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", - "WARNING:tensorflow:Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", - "WARNING: Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", - "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1288: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n", + "6 6\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/backend.py:435: UserWarning: `tf.keras.backend.set_learning_phase` is deprecated and will be removed after 2020-10-11. To update it, simply pass a True/False value to the `training` argument of the `__call__` method of your layer or model.\n", + " warnings.warn('`tf.keras.backend.set_learning_phase` is deprecated and '\n", + "WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/array_ops.py:5049: calling gather (from tensorflow.python.ops.array_ops) with validate_indices is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", - "Call initializer instance with the dtype argument instead of passing it to the constructor\n", - "WARNING:tensorflow:Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AttributeError: module 'gast' has no attribute 'Num'\n", - "WARNING: Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AttributeError: module 'gast' has no attribute 'Num'\n", - "WARNING:tensorflow:Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", - "WARNING: Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", - "Epoch 1/24\n", - "988129/988129 [==============================] - 22s 23us/sample - loss: 4.0715\n", - "Epoch 2/24\n", - "988129/988129 [==============================] - 21s 21us/sample - loss: 3.7855\n", - "Epoch 3/24\n", - "988129/988129 [==============================] - 21s 22us/sample - loss: 3.5596\n", - "Epoch 4/24\n", - "988129/988129 [==============================] - 21s 21us/sample - loss: 3.4199\n", - "Epoch 5/24\n", - "988129/988129 [==============================] - 21s 22us/sample - loss: 3.3216\n", - "Epoch 6/24\n", - "988129/988129 [==============================] - 21s 21us/sample - loss: 3.2499\n", - "Epoch 7/24\n", - "988129/988129 [==============================] - 21s 22us/sample - loss: 3.1846\n", - "Epoch 8/24\n", - "988129/988129 [==============================] - 21s 22us/sample - loss: 3.1264\n", - "Epoch 9/24\n", - "988129/988129 [==============================] - 22s 22us/sample - loss: 3.0923\n", - "Epoch 10/24\n", - "988129/988129 [==============================] - 21s 22us/sample - loss: 3.0566\n", - "Epoch 11/24\n", - "988129/988129 [==============================] - 21s 22us/sample - loss: 3.0336\n", - "Epoch 12/24\n", - "988129/988129 [==============================] - 21s 22us/sample - loss: 3.0066\n", - "Epoch 13/24\n", - "988129/988129 [==============================] - 21s 22us/sample - loss: 2.9872\n", - "Epoch 14/24\n", - "988129/988129 [==============================] - 21s 22us/sample - loss: 2.9659\n", - "Epoch 15/24\n", - "988129/988129 [==============================] - 21s 22us/sample - loss: 2.9476\n", - "Epoch 16/24\n", - "988129/988129 [==============================] - 21s 22us/sample - loss: 2.9363\n", - "Epoch 17/24\n", - "988129/988129 [==============================] - 21s 21us/sample - loss: 2.9267\n", - "Epoch 18/24\n", - "988129/988129 [==============================] - 21s 22us/sample - loss: 2.9179\n", - "Epoch 19/24\n", - "988129/988129 [==============================] - 21s 22us/sample - loss: 2.9012\n", - "Epoch 20/24\n", - "988129/988129 [==============================] - 21s 21us/sample - loss: 2.8925\n", - "Epoch 21/24\n", - "988129/988129 [==============================] - 21s 21us/sample - loss: 2.8830\n", - "Epoch 22/24\n", - "988129/988129 [==============================] - 22s 22us/sample - loss: 2.8797\n", - "Epoch 23/24\n", - "988129/988129 [==============================] - 21s 22us/sample - loss: 2.8711\n", - "Epoch 24/24\n", - "988129/988129 [==============================] - 21s 22us/sample - loss: 2.8619\n", + "The `validate_indices` argument has no effect. Indices are always validated on CPU and never validated on GPU.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train on 988129 samples\n", + "Epoch 1/20\n", + "988129/988129 [==============================] - 24s 25us/sample - loss: 4.4995\n", + "Epoch 2/20\n", + "988129/988129 [==============================] - 23s 23us/sample - loss: 4.2307\n", + "Epoch 3/20\n", + "988129/988129 [==============================] - 25s 25us/sample - loss: 3.8902\n", + "Epoch 4/20\n", + "988129/988129 [==============================] - 24s 24us/sample - loss: 3.6825\n", + "Epoch 5/20\n", + "988129/988129 [==============================] - 23s 23us/sample - loss: 3.5604\n", + "Epoch 6/20\n", + "988129/988129 [==============================] - 23s 23us/sample - loss: 3.4642\n", + "Epoch 7/20\n", + "988129/988129 [==============================] - 23s 23us/sample - loss: 3.3803\n", + "Epoch 8/20\n", + "988129/988129 [==============================] - 23s 23us/sample - loss: 3.3126\n", + "Epoch 9/20\n", + "988129/988129 [==============================] - 23s 23us/sample - loss: 3.2583\n", + "Epoch 10/20\n", + "988129/988129 [==============================] - 23s 23us/sample - loss: 3.2177\n", + "Epoch 11/20\n", + "988129/988129 [==============================] - 23s 23us/sample - loss: 3.1791\n", + "Epoch 12/20\n", + "988129/988129 [==============================] - 23s 23us/sample - loss: 3.1472\n", + "Epoch 13/20\n", + "988129/988129 [==============================] - 23s 23us/sample - loss: 3.1246\n", + "Epoch 14/20\n", + "988129/988129 [==============================] - 23s 23us/sample - loss: 3.0992\n", + "Epoch 15/20\n", + "988129/988129 [==============================] - 24s 24us/sample - loss: 3.0796\n", + "Epoch 16/20\n", + "988129/988129 [==============================] - 23s 23us/sample - loss: 3.0601\n", + "Epoch 17/20\n", + "988129/988129 [==============================] - 23s 23us/sample - loss: 3.0418\n", + "Epoch 18/20\n", + "988129/988129 [==============================] - 23s 23us/sample - loss: 3.0265\n", + "Epoch 19/20\n", + "988129/988129 [==============================] - 23s 23us/sample - loss: 3.0119\n", + "Epoch 20/20\n", + "988129/988129 [==============================] - 23s 23us/sample - loss: 2.9994\n", "(6040, 32)\n", "(3706, 32)\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py:2426: UserWarning: `Model.state_updates` will be removed in a future version. This property should not be used in TensorFlow 2.0, as `updates` are applied automatically.\n", + " warnings.warn('`Model.state_updates` will be removed in a future version. '\n" + ] } ], "source": [ @@ -342,13 +271,19 @@ "\n", "# 1.Label Encoding for sparse features,and process sequence features with `gen_date_set` and `gen_model_input`\n", "\n", - "features = ['user_id', 'movie_id', 'gender', 'age', 'occupation', 'zip']\n", + "features = ['user_id', 'gender', 'age', 'occupation', 'zip']\n", "feature_max_idx = {}\n", "for feature in features:\n", " lbe = LabelEncoder()\n", " data[feature] = lbe.fit_transform(data[feature]) + 1\n", " feature_max_idx[feature] = data[feature].max() + 1\n", "\n", + "id_count = data['movie_id'].value_counts() \n", + "mapdict = {t[0]: i for i, t in\n", + " enumerate(sorted([(k, v) for k, v in id_count.to_dict().items()], key=lambda x: x[1], reverse=True))}\n", + "data['movie_id'] = data['movie_id'].map(mapdict)\n", + "feature_max_idx['movie_id'] = data['movie_id'].max() + 1\n", + "\n", "user_profile = data[[\"user_id\", \"gender\", \"age\", \"occupation\", \"zip\"]].drop_duplicates('user_id')\n", "\n", "item_profile = data[[\"movie_id\"]].drop_duplicates('movie_id')\n", @@ -379,14 +314,14 @@ "\n", "# 3.Define Model and train\n", "\n", - "K.set_learning_phase(True)\n", - "\n", "import tensorflow as tf\n", "if tf.__version__ >= '2.0.0':\n", " tf.compat.v1.disable_eager_execution()\n", - "\n", + "else:\n", + " K.set_learning_phase(True)\n", + " \n", "model = YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=100, user_dnn_hidden_units=(128,64, embedding_dim))\n", - "# model = MIND(user_feature_columns,item_feature_columns,dynamic_k=False,p=1,k_max=2,num_sampled=100,user_dnn_hidden_units=(128,64, embedding_dim))\n", + "# model = MIND(user_feature_columns,item_feature_columns,dynamic_k=True,k_max=2,num_sampled=100,user_dnn_hidden_units=(128,64, embedding_dim))\n", "\n", "model.compile(optimizer=\"adam\", loss=sampledsoftmaxloss) # \"binary_crossentropy\")\n", "\n", @@ -412,7 +347,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "w_G3KWslKmJo" }, "source": [ @@ -422,30 +356,27 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "5SvyQLNVKkcs" }, "source": [] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": 5, "metadata": { "colab": { - "base_uri": "https://localhost:8080/", - "height": 51 + "base_uri": "https://localhost:8080/" }, - "colab_type": "code", "id": "j2ZNYNBOOqrN", - "outputId": "2938673c-ff81-49a2-86d8-4266d2060ea3" + "outputId": "442609e8-f94d-42c3-d945-582374a5fa77" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Requirement already satisfied: faiss-cpu in /usr/local/lib/python3.6/dist-packages (1.6.3)\n", - "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from faiss-cpu) (1.18.2)\n" + "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", + "Requirement already satisfied: faiss-cpu in /usr/local/lib/python3.7/dist-packages (1.7.2)\n" ] } ], @@ -455,22 +386,20 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": 6, "metadata": { "colab": { - "base_uri": "https://localhost:8080/", - "height": 85 + "base_uri": "https://localhost:8080/" }, - "colab_type": "code", "id": "6TY1l27iJU8U", - "outputId": "5316c37f-fef1-44b3-8c31-6600d1e44da5" + "outputId": "3070ad94-9f84-4b51-d095-18053b84f5ce" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "6040it [00:01, 4290.79it/s]" + "6040it [00:02, 3004.17it/s]" ] }, { @@ -478,8 +407,8 @@ "output_type": "stream", "text": [ "\n", - "recall 0.26473509933774836\n", - "hit rate 0.26473509933774836\n" + "recall 0.3033112582781457\n", + "hit rate 0.3033112582781457\n" ] }, { @@ -524,10 +453,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": 6, "metadata": { - "colab": {}, - "colab_type": "code", "id": "a97TB0obOrRe" }, "outputs": [], @@ -537,13 +464,12 @@ "metadata": { "accelerator": "GPU", "colab": { - "authorship_tag": "ABX9TyOrqZNeC0DgyPX2JmJid1m7", "collapsed_sections": [], - "include_colab_link": true, "name": "colab_MovieLen1M_YoutubeDNN", "provenance": [], "toc_visible": true }, + "gpuClass": "standard", "kernelspec": { "display_name": "Python 3", "language": "python", diff --git a/examples/run_sdm.py b/examples/run_sdm.py index fdf1def..902ea17 100644 --- a/examples/run_sdm.py +++ b/examples/run_sdm.py @@ -18,13 +18,19 @@ # 1.Label Encoding for sparse features,and process sequence features with `gen_date_set` and `gen_model_input` - features = ['user_id', 'movie_id', 'gender', 'age', 'occupation', 'zip', 'genres'] + features = ['user_id', 'gender', 'age', 'occupation', 'zip'] feature_max_idx = {} for feature in features: lbe = LabelEncoder() data[feature] = lbe.fit_transform(data[feature]) + 1 feature_max_idx[feature] = data[feature].max() + 1 + id_count = data['movie_id'].value_counts() + mapdict = {t[0]: i for i, t in + enumerate(sorted([(k, v) for k, v in id_count.to_dict().items()], key=lambda x: x[1], reverse=True))} + data['movie_id'] = data['movie_id'].map(mapdict) + feature_max_idx['movie_id'] = data['movie_id'].max() + 1 + user_profile = data[["user_id", "gender", "age", "occupation", "zip", "genres"]].drop_duplicates('user_id') item_profile = data[["movie_id"]].drop_duplicates('movie_id') diff --git a/examples/run_youtubednn.py b/examples/run_youtubednn.py index d98a472..b9881ed 100644 --- a/examples/run_youtubednn.py +++ b/examples/run_youtubednn.py @@ -1,13 +1,12 @@ import pandas as pd from deepctr.feature_column import SparseFeat, VarLenSparseFeat +from deepmatch.models import * +from deepmatch.utils import sampledsoftmaxloss from preprocess import gen_data_set, gen_model_input from sklearn.preprocessing import LabelEncoder from tensorflow.python.keras import backend as K from tensorflow.python.keras.models import Model -from deepmatch.models import * -from deepmatch.utils import sampledsoftmaxloss - if __name__ == "__main__": data = pd.read_csvdata = pd.read_csv("./movielens_sample.txt") @@ -17,13 +16,19 @@ # 1.Label Encoding for sparse features,and process sequence features with `gen_date_set` and `gen_model_input` - features = ['user_id', 'movie_id', 'gender', 'age', 'occupation', 'zip'] + features = ['user_id', 'gender', 'age', 'occupation', 'zip'] feature_max_idx = {} for feature in features: lbe = LabelEncoder() data[feature] = lbe.fit_transform(data[feature]) + 1 feature_max_idx[feature] = data[feature].max() + 1 + id_count = data['movie_id'].value_counts() + mapdict = {t[0]: i for i, t in + enumerate(sorted([(k, v) for k, v in id_count.to_dict().items()], key=lambda x: x[1], reverse=True))} + data['movie_id'] = data['movie_id'].map(mapdict) + feature_max_idx['movie_id'] = data['movie_id'].max() + 1 + user_profile = data[["user_id", "gender", "age", "occupation", "zip"]].drop_duplicates('user_id') item_profile = data[["movie_id"]].drop_duplicates('movie_id') @@ -54,13 +59,15 @@ # 3.Define Model and train - K.set_learning_phase(True) import tensorflow as tf + if tf.__version__ >= '2.0.0': - tf.compat.v1.disable_eager_execution() + tf.compat.v1.disable_eager_execution() + else: + K.set_learning_phase(True) model = YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, user_dnn_hidden_units=(64, embedding_dim)) - #model = MIND(user_feature_columns,item_feature_columns,dynamic_k=False,p=1,k_max=2,num_sampled=5,user_dnn_hidden_units=(64, embedding_dim)) + # model = MIND(user_feature_columns,item_feature_columns,dynamic_k=True,k_max=2,num_sampled=5,user_dnn_hidden_units=(64, embedding_dim)) model.compile(optimizer="adam", loss=sampledsoftmaxloss) # "binary_crossentropy") diff --git a/setup.py b/setup.py index 8ab2c5c..d5fe36e 100644 --- a/setup.py +++ b/setup.py @@ -4,14 +4,14 @@ long_description = fh.read() REQUIRED_PACKAGES = [ - 'h5py', 'requests', "deepctr==0.8.2" + 'requests', "deepctr==0.9.1" ] setuptools.setup( name="deepmatch", - version="0.2.0", + version="0.2.1", author="Weichen Shen", - author_email="wcshen1994@163.com", + author_email="weichenswc@163.com", description="Deep matching model library for recommendations, advertising. It's easy to train models and to **export representation vectors** for user and item which can be used for **ANN search**.", long_description=long_description, long_description_content_type="text/markdown", @@ -38,6 +38,7 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Topic :: Scientific/Engineering', 'Topic :: Scientific/Engineering :: Artificial Intelligence', 'Topic :: Software Development', diff --git a/tests/models/MIND_test.py b/tests/models/MIND_test.py index 0b9509b..ea9c9a1 100644 --- a/tests/models/MIND_test.py +++ b/tests/models/MIND_test.py @@ -1,21 +1,28 @@ +import pytest import tensorflow as tf -from tensorflow.python.keras import backend as K - from deepmatch.models import MIND from deepmatch.utils import sampledsoftmaxloss +from tensorflow.python.keras import backend as K + from ..utils import check_model, get_xy_fd -def test_MIND(): +@pytest.mark.parametrize( + 'dynamic_k,p', + [(False, 1), (True, 100) + ] +) +def test_MIND(dynamic_k, p): model_name = "MIND" x, y, user_feature_columns, item_feature_columns = get_xy_fd(False) - K.set_learning_phase(True) if tf.__version__ >= '2.0.0': tf.compat.v1.disable_eager_execution() - - model = MIND(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) + else: + K.set_learning_phase(True) + model = MIND(user_feature_columns, item_feature_columns, num_sampled=2, p=p, dynamic_k=dynamic_k, + user_dnn_hidden_units=(16, 4)) model.compile('adam', sampledsoftmaxloss) check_model(model, model_name, x, y) diff --git a/tests/models/SDM_test.py b/tests/models/SDM_test.py index 60e3e5a..072caf8 100644 --- a/tests/models/SDM_test.py +++ b/tests/models/SDM_test.py @@ -9,13 +9,12 @@ def test_SDM(): model_name = "SDM" - tf.keras.backend.set_learning_phase(1) x, y, user_feature_columns, item_feature_columns, history_feature_list = get_xy_fd_sdm(False) - K.set_learning_phase(True) if tf.__version__ >= '2.0.0': tf.compat.v1.disable_eager_execution() - + else: + K.set_learning_phase(True) model = SDM(user_feature_columns, item_feature_columns, history_feature_list, units=8) # model.summary() diff --git a/tests/models/YoutubeDNN_test.py b/tests/models/YoutubeDNN_test.py index 46afb03..a014c16 100644 --- a/tests/models/YoutubeDNN_test.py +++ b/tests/models/YoutubeDNN_test.py @@ -10,10 +10,11 @@ def test_YoutubeDNN(): model_name = "YoutubeDNN" x, y, user_feature_columns, item_feature_columns = get_xy_fd(False) - K.set_learning_phase(True) if tf.__version__ >= '2.0.0': tf.compat.v1.disable_eager_execution() + else: + K.set_learning_phase(True) model = YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) model.compile('adam', sampledsoftmaxloss)