Update feature config generation method & Fix bugs

- Adopted a new feature config generation method
- Simplified input mechanism for multi-valued features.
- Fix bug: in some cases the order of model input placeholder and actual model input list does not match.
- Fix bug: Divide by zero overflow in `SequencePoolingLayer` when sequence length is 0.

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2018 Weichen Shen 
+Copyright (c) 2018-2019 Weichen Shen 
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

deepctr/__init__.py

@@ -2,6 +2,6 @@
 from .import layers
 from .import sequence
 from . import models
-from .utils import check_version
-__version__ = '0.2.2'
+from .utils import check_version, SingleFeat, VarLenFeat
+__version__ = '0.2.3'
 check_version(__version__)

deepctr/input_embedding.py

@@ -1,75 +1,85 @@
+from collections import OrderedDict
 from itertools import chain
 
-from tensorflow.python.keras import Input
 from tensorflow.python.keras.initializers import RandomNormal
-from tensorflow.python.keras.layers import Embedding, Dense, Reshape, Concatenate
+from tensorflow.python.keras.layers import Embedding, Dense, Reshape, Concatenate, Input, add
 from tensorflow.python.keras.regularizers import l2
 from .sequence import SequencePoolingLayer
-from .utils import get_linear_logit
 
 
 def create_input_dict(feature_dim_dict, prefix=''):
- sparse_input = {feat: Input(shape=(1,), name=prefix+'sparse_' + str(i) + '-' + feat) for i, feat in
- enumerate(feature_dim_dict["sparse"])}
- dense_input = {feat: Input(shape=(1,), name=prefix+'dense_' + str(i) + '-' + feat) for i, feat in
- enumerate(feature_dim_dict["dense"])}
+ sparse_input = OrderedDict()
+ for i, feat in enumerate(feature_dim_dict["sparse"]):
+ sparse_input[feat.name] = Input(
+ shape=(1,), name=prefix+'sparse_' + str(i) + '-' + feat.name)
+
+ dense_input = OrderedDict()
+
+ for i, feat in enumerate(feature_dim_dict["dense"]):
+ dense_input[feat] = Input(
+ shape=(1,), name=prefix+'dense_' + str(i) + '-' + feat.name)
+
  return sparse_input, dense_input
 
 
-def create_sequence_input_dict(feature_dim_dict):
+def create_sequence_input_dict(feature_dim_dict, mask_zero=True):
 
  sequence_dim_dict = feature_dim_dict.get('sequence', [])
  sequence_input_dict = {feat.name: Input(shape=(feat.maxlen,), name='seq_' + str(
  i) + '-' + feat.name) for i, feat in enumerate(sequence_dim_dict)}
  sequence_pooling_dict = {feat.name: feat.combiner
  for i, feat in enumerate(sequence_dim_dict)}
- sequence_len_dict = {feat.name: Input(shape=(
- 1,), name='seq_length'+str(i)+'-'+feat.name) for i, feat in enumerate(sequence_dim_dict)}
- sequence_max_len_dict = {feat.name: feat.maxlen
- for i, feat in enumerate(sequence_dim_dict)}
+ if mask_zero:
+ sequence_len_dict, sequence_max_len_dict = None, None
+ else:
+ sequence_len_dict = {feat.name: Input(shape=(
+ 1,), name='seq_length'+str(i)+'-'+feat.name) for i, feat in enumerate(sequence_dim_dict)}
+ sequence_max_len_dict = {feat.name: feat.maxlen
+ for i, feat in enumerate(sequence_dim_dict)}
  return sequence_input_dict, sequence_pooling_dict, sequence_len_dict, sequence_max_len_dict
 
 
-def create_embedding_dict(feature_dim_dict, embedding_size, init_std, seed, l2_reg, prefix='sparse'):
+def create_embedding_dict(feature_dim_dict, embedding_size, init_std, seed, l2_reg, prefix='sparse', seq_mask_zero=True):
  if embedding_size == 'auto':
 
- sparse_embedding = {feat: Embedding(feature_dim_dict["sparse"][feat], 6 * int(pow(feature_dim_dict["sparse"][feat], 0.25)),
- embeddings_initializer=RandomNormal(
+ sparse_embedding = {feat.name: Embedding(feat.dimension, 6 * int(pow(feat.dimension, 0.25)),
+  embeddings_initializer=RandomNormal(
  mean=0.0, stddev=init_std, seed=seed),
  embeddings_regularizer=l2(l2_reg),
- name=prefix+'_emb_' + str(i) + '-' + feat) for i, feat in
+ name=prefix+'_emb_' + str(i) + '-' + feat.name) for i, feat in
  enumerate(feature_dim_dict["sparse"])}
  else:
 
- sparse_embedding = {feat: Embedding(feature_dim_dict["sparse"][feat], embedding_size,
- embeddings_initializer=RandomNormal(
- mean=0.0, stddev=init_std, seed=seed),
- embeddings_regularizer=l2(l2_reg),
- name=prefix+'_emb_' + str(i) + '-' + feat) for i, feat in
- enumerate(feature_dim_dict["sparse"])}
+ sparse_embedding = {feat.name: Embedding(feat.dimension, embedding_size,
+ embeddings_initializer=RandomNormal(
+ mean=0.0, stddev=init_std, seed=seed),
+ embeddings_regularizer=l2(
+ l2_reg),
+ name=prefix+'_emb_' + str(i) + '-' + feat.name) for i, feat in
+ enumerate(feature_dim_dict["sparse"])}
 
  if 'sequence' in feature_dim_dict:
  count = len(sparse_embedding)
  sequence_dim_list = feature_dim_dict['sequence']
  for feat in sequence_dim_list:
- if feat.name not in sparse_embedding:
-  if embedding_size == "auto":
-  sparse_embedding[feat.name] = Embedding(feat.dimension, 6 * int(pow(feat.dimension, 0.25)),
-  embeddings_initializer=RandomNormal(
-  mean=0.0, stddev=init_std, seed=seed),
-  embeddings_regularizer=l2(
-  l2_reg),
-  name=prefix + '_emb_' + str(count) + '-' + feat.name)
-
-  else:
-  sparse_embedding[feat.name] = Embedding(feat.dimension, embedding_size,
-  embeddings_initializer=RandomNormal(
-  mean=0.0, stddev=init_std, seed=seed),
-  embeddings_regularizer=l2(
-  l2_reg),
-  name=prefix+'_emb_' + str(count) + '-' + feat.name)
-
-  count += 1
+ # if feat.name not in sparse_embedding:
+ if embedding_size == "auto":
+ sparse_embedding[feat.name] = Embedding(feat.dimension, 6 * int(pow(feat.dimension, 0.25)),
+ embeddings_initializer=RandomNormal(
+ mean=0.0, stddev=init_std, seed=seed),
+ embeddings_regularizer=l2(
+ l2_reg),
+ name=prefix + '_emb_' + str(count) + '-' + feat.name, mask_zero=seq_mask_zero)
+
+ else:
+ sparse_embedding[feat.name] = Embedding(feat.dimension, embedding_size,
+ embeddings_initializer=RandomNormal(
+ mean=0.0, stddev=init_std, seed=seed),
+ embeddings_regularizer=l2(
+ l2_reg),
+ name=prefix+'_emb_' + str(count) + '-' + feat.name, mask_zero=seq_mask_zero)
+
+ count += 1
 
  return sparse_embedding
 
@@ -109,7 +119,6 @@ def merge_sequence_input(embedding_dict, embed_list, sequence_input_dict, sequen
 
 
 def get_embedding_vec_list(embedding_dict, input_dict):
-
  return [embedding_dict[feat](v)
  for feat, v in input_dict.items()]
 
@@ -121,12 +130,15 @@ def get_varlen_embedding_vec_dict(embedding_dict, input_dict):
 
 
 def get_pooling_vec_list(sequence_embed_dict, sequence_len_dict, sequence_max_len_dict, sequence_pooling_dict):
- return [SequencePoolingLayer(sequence_max_len_dict[feat], sequence_pooling_dict[feat])(
- [v, sequence_len_dict[feat]]) for feat, v in sequence_embed_dict.items()]
+ if sequence_max_len_dict is None or sequence_len_dict is None:
+ return [SequencePoolingLayer(-1, sequence_pooling_dict[feat])(v) for feat, v in sequence_embed_dict.items()]
+ else:
+ return [SequencePoolingLayer(sequence_max_len_dict[feat], sequence_pooling_dict[feat])(
+ [v, sequence_len_dict[feat]]) for feat, v in sequence_embed_dict.items()]
 
 
 def get_inputs_list(inputs):
- return list(chain(*list(map(lambda x: x.values(), inputs))))
+ return list(chain(*list(map(lambda x: x.values(), filter(lambda x: x is not None, inputs)))))
 
 
 def get_inputs_embedding(feature_dim_dict, embedding_size, l2_reg_embedding, l2_reg_linear, init_std, seed, include_linear=True):
@@ -162,3 +174,25 @@ def get_inputs_embedding(feature_dim_dict, embedding_size, l2_reg_embedding, l2_
  inputs_list = get_inputs_list(
  [sparse_input_dict, dense_input_dict, sequence_input_dict, sequence_input_len_dict])
  return deep_emb_list, linear_logit, inputs_list
+
+
+def get_linear_logit(linear_term, dense_input_, l2_reg):
+ if len(linear_term) > 1:
+ linear_term = add(linear_term)
+ elif len(linear_term) == 1:
+ linear_term = linear_term[0]
+ else:
+ linear_term = None
+
+ dense_input = list(dense_input_.values())
+ if len(dense_input) > 0:
+ dense_input__ = dense_input[0] if len(
+ dense_input) == 1 else Concatenate()(dense_input)
+ linear_dense_logit = Dense(
+ 1, activation=None, use_bias=False, kernel_regularizer=l2(l2_reg))(dense_input__)
+ if linear_term is not None:
+ linear_term = add([linear_dense_logit, linear_term])
+ else:
+ linear_term = linear_dense_logit
+
+ return linear_term

deepctr/layers.py

@@ -1,7 +1,7 @@
 import itertools
 from tensorflow.python.keras.layers import Layer, BatchNormalization
 from tensorflow.python.keras.regularizers import l2
-from tensorflow.python.keras.initializers import Zeros, glorot_normal, glorot_uniform
+from tensorflow.python.keras.initializers import Zeros, Ones, glorot_normal, glorot_uniform
 from tensorflow.python.keras import backend as K
 import tensorflow as tf
 from .activations import activation_fun
@@ -355,10 +355,9 @@ def call(self, inputs, **kwargs):
  x_0 = tf.expand_dims(inputs, axis=2)
  x_l = x_0
  for i in range(self.layer_num):
- xl_w = tf.tensordot(tf.transpose(
- x_l, [0, 2, 1]), self.kernels[i], axes=(-1, 0))
+ xl_w = tf.tensordot(x_l, self.kernels[i], axes=(1, 0))
  dot_ = tf.matmul(x_0, xl_w)
- x_l = dot_ + x_l + self.bias[i]
+ x_l = dot_ + self.bias[i] + x_l
  x_l = tf.squeeze(x_l, axis=2)
  return x_l
 
@@ -504,7 +503,6 @@ def get_config(self,):
  return dict(list(base_config.items()) + list(config.items()))
 
 
-
 class InteractingLayer(Layer):
  """A Layer used in AutoInt that model the correlations between different feature fields by multi-head self-attention mechanism.
 
@@ -524,6 +522,7 @@ class InteractingLayer(Layer):
  References
  - [Song W, Shi C, Xiao Z, et al. AutoInt: Automatic Feature Interaction Learning via Self-Attentive Neural Networks[J]. arXiv preprint arXiv:1810.11921, 2018.](https://arxiv.org/abs/1810.11921)
  """
+
  def __init__(self, att_embedding_size=8, head_num=2, use_res=True, seed=1024, **kwargs):
  if head_num <= 0:
  raise ValueError('head_num must be a int > 0')
@@ -535,7 +534,8 @@ def __init__(self, att_embedding_size=8, head_num=2, use_res=True, seed=1024, **
 
  def build(self, input_shape):
  if len(input_shape) != 3:
- raise ValueError("Unexpected inputs dimensions %d, expect to be 3 dimensions" % (len(input_shape)))
+ raise ValueError(
+ "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (len(input_shape)))
  embedding_size = input_shape[-1].value
  self.W_Query = self.add_weight(name='query', shape=[embedding_size, self.att_embedding_size * self.head_num], dtype=tf.float32,
  initializer=tf.keras.initializers.glorot_uniform(seed=self.seed))
@@ -547,26 +547,32 @@ def build(self, input_shape):
  self.W_Res = self.add_weight(name='res', shape=[embedding_size, self.att_embedding_size * self.head_num], dtype=tf.float32,
  initializer=tf.keras.initializers.glorot_uniform(seed=self.seed))
 
- super(InteractingLayer, self).build(input_shape) # Be sure to call this somewhere!
+ # Be sure to call this somewhere!
+ super(InteractingLayer, self).build(input_shape)
 
  def call(self, inputs, **kwargs):
  if K.ndim(inputs) != 3:
- raise ValueError("Unexpected inputs dimensions %d, expect to be 3 dimensions" % (K.ndim(inputs)))
+ raise ValueError(
+ "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (K.ndim(inputs)))
 
- querys = tf.tensordot(inputs, self.W_Query, axes=(-1, 0)) # None F D*head_num
+ querys = tf.tensordot(inputs, self.W_Query,
+ axes=(-1, 0)) # None F D*head_num
  keys = tf.tensordot(inputs, self.W_key, axes=(-1, 0))
  values = tf.tensordot(inputs, self.W_Value, axes=(-1, 0))
 
- querys = tf.stack(tf.split(querys, self.head_num, axis=2)) # head_num None F D
+ # head_num None F D
+ querys = tf.stack(tf.split(querys, self.head_num, axis=2))
  keys = tf.stack(tf.split(keys, self.head_num, axis=2))
  values = tf.stack(tf.split(values, self.head_num, axis=2))
 
- inner_product = tf.matmul(querys, keys, transpose_b=True) # head_num None F F
+ inner_product = tf.matmul(
+ querys, keys, transpose_b=True) # head_num None F F
  self.normalized_att_scores = tf.nn.softmax(inner_product)
 
- result = tf.matmul(self.normalized_att_scores, values)#head_num None F D
+ result = tf.matmul(self.normalized_att_scores,
+ values) # head_num None F D
  result = tf.concat(tf.split(result, self.head_num, ), axis=-1)
- result = tf.squeeze(result, axis=0)#None F D*head_num
+ result = tf.squeeze(result, axis=0) # None F D*head_num
 
  if self.use_res:
  result += tf.tensordot(inputs, self.W_Res, axes=(-1, 0))
@@ -648,7 +654,7 @@ def build(self, input_shape):
  super(LocalActivationUnit, self).build(
  input_shape) # Be sure to call this somewhere!
 
- def call(self, inputs, **kwargs):
+ def call(self, inputs, training=None, **kwargs):
 
  query, keys = inputs
 
@@ -657,7 +663,7 @@ def call(self, inputs, **kwargs):
 
  att_input = tf.concat(
  [queries, keys, queries - keys, queries * keys], axis=-1)
- att_input = tf.layers.batch_normalization(att_input)
+
  att_out = MLP(self.hidden_size, self.activation, self.l2_reg,
  self.keep_prob, self.use_bn, seed=self.seed)(att_input)
  attention_score = tf.nn.bias_add(tf.tensordot(
@@ -724,7 +730,8 @@ def build(self, input_shape):
 
  super(MLP, self).build(input_shape) # Be sure to call this somewhere!
 
- def call(self, inputs, **kwargs):
+ def call(self, inputs, training=None, **kwargs):
+
  deep_input = inputs
 
  for i in range(len(self.hidden_size)):
@@ -734,9 +741,10 @@ def call(self, inputs, **kwargs):
  # kernel_initializer=glorot_normal(seed=self.seed), \
  # kernel_regularizer=l2(self.l2_reg))(deep_input)
  if self.use_bn:
- fc = BatchNormalization()(fc)
+ fc = tf.keras.layers.BatchNormalization()(fc)
  fc = activation_fun(self.activation, fc)
- fc = tf.nn.dropout(fc, self.keep_prob)
+ #fc = tf.nn.dropout(fc, self.keep_prob)
+ fc = tf.keras.layers.Dropout(1 - self.keep_prob)(fc,)
  deep_input = fc
 
  return deep_input
@@ -901,7 +909,7 @@ class PredictionLayer(Layer):
  Arguments
  - **activation**: Activation function to use.
 
- - **use_bias**: bool.Whther add bias term.
+ - **use_bias**: bool.Whether add bias term or not.
  """
 
  def __init__(self, activation='sigmoid', use_bias=True, **kwargs):
@@ -933,4 +941,4 @@ def compute_output_shape(self, input_shape):
  def get_config(self,):
  config = {'activation': self.activation, 'use_bias': self.use_bias}
  base_config = super(PredictionLayer, self).get_config()
- return dict(list(base_config.items()) + list(config.items()))
+ return dict(list(base_config.items()) + list(config.items()))

deepctr/models/afm.py

@@ -12,7 +12,7 @@
 import tensorflow as tf
 from ..input_embedding import get_inputs_embedding
 from ..layers import PredictionLayer, AFMLayer, FM
-from ..utils import concat_fun
+from ..utils import concat_fun, check_feature_config_dict
 
 
 def AFM(feature_dim_dict, embedding_size=8, use_attention=True, attention_factor=8,
@@ -34,21 +34,12 @@ def AFM(feature_dim_dict, embedding_size=8, use_attention=True, attention_factor
  :return: A Keras model instance.
  """
 
- if not isinstance(feature_dim_dict,
- dict) or "sparse" not in feature_dim_dict or "dense" not in feature_dim_dict:
- raise ValueError(
- "feature_dim_dict must be a dict like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_4','field_5']}")
- if not isinstance(feature_dim_dict["sparse"], dict):
- raise ValueError("feature_dim_dict['sparse'] must be a dict,cur is", type(
- feature_dim_dict['sparse']))
- if not isinstance(feature_dim_dict["dense"], list):
- raise ValueError("feature_dim_dict['dense'] must be a list,cur is", type(
- feature_dim_dict['dense']))
+ check_feature_config_dict(feature_dim_dict)
 
  deep_emb_list, linear_logit, inputs_list = get_inputs_embedding(
  feature_dim_dict, embedding_size, l2_reg_embedding, l2_reg_linear, init_std, seed)
 
- fm_input = concat_fun(deep_emb_list,axis=1)
+ fm_input = concat_fun(deep_emb_list, axis=1)
  if use_attention:
  fm_logit = AFMLayer(attention_factor, l2_reg_att,
  keep_prob, seed)(deep_emb_list)

deepctr/models/autoint.py

@@ -12,7 +12,7 @@
 import tensorflow as tf
 from ..input_embedding import get_inputs_embedding
 from ..layers import PredictionLayer, MLP, InteractingLayer
-from ..utils import concat_fun
+from ..utils import concat_fun, check_feature_config_dict
 
 
 def AutoInt(feature_dim_dict, embedding_size=8, att_layer_num=3, att_embedding_size=8, att_head_num=2, att_res=True, hidden_size=(256, 256), activation='relu',
@@ -40,9 +40,7 @@ def AutoInt(feature_dim_dict, embedding_size=8, att_layer_num=3, att_embedding_s
 
  if len(hidden_size) <= 0 and att_layer_num <= 0:
  raise ValueError("Either hidden_layer or att_layer_num must > 0")
- if not isinstance(feature_dim_dict, dict) or "sparse" not in feature_dim_dict or "dense" not in feature_dim_dict:
- raise ValueError(
- "feature_dim must be a dict like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_5',]}")
+ check_feature_config_dict(feature_dim_dict)
 
  deep_emb_list, _, inputs_list = get_inputs_embedding(
  feature_dim_dict, embedding_size, l2_reg_embedding, 0, init_std, seed, False)