1.feature.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from tqdm import tqdm\n",
    "import lightgbm as lgb\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "import warnings\n",
    "from sklearn.metrics import f1_score, classification_report\n",
    "import gc\n",
    "import xgboost as xgb\n",
    "from scipy import stats\n",
    "import datetime\n",
    "import time\n",
    "from scipy.stats import entropy, kurtosis\n",
    "import multiprocessing\n",
    "from gensim.models.word2vec import LineSentence\n",
    "from gensim.corpora import WikiCorpus\n",
    "from gensim.models import Word2Vec\n",
    "tqdm.pandas()\n",
    "\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', None)\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "current_path = './'\n",
    "seed = 2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(os.path.join(current_path, 'raw_data', 'train.csv'))\n",
    "df_test = pd.read_csv(os.path.join(current_path, 'raw_data', 'test.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feature = pd.concat([df_train, df_test], sort=False)\n",
    "df_feature = df_feature.sort_values(\n",
    "    ['deviceid', 'ts']).reset_index().drop('index', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feature['newsid'] = df_feature['newsid'].map(lambda x: str(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 时间\n",
    "df_feature['ts_datetime'] = df_feature['ts'] + 8 * 60 * 60 * 1000\n",
    "df_feature['ts_datetime'] = pd.to_datetime(\n",
    "    df_feature['ts_datetime'], unit='ms')\n",
    "df_feature['day'] = df_feature['ts_datetime'].dt.day\n",
    "df_feature['hour'] = df_feature['ts_datetime'].dt.hour\n",
    "df_feature['minute'] = df_feature['ts_datetime'].dt.minute\n",
    "df_feature['minute10'] = (df_feature['minute'] // 10) * 10\n",
    "\n",
    "df_feature['hourl'] = df_feature['day'] * 24 + df_feature['hour']\n",
    "df_feature['hourl'] = df_feature['hourl'] - df_feature['hourl'].min()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 基本特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "group = df_feature.groupby('deviceid')\n",
    "df_feature['ts_before'] = group['ts'].shift(0) - group['ts'].shift(1)\n",
    "df_feature['ts_before'] = df_feature['ts_before'].fillna(3 * 60 * 1000)\n",
    "INDEX = df_feature[df_feature['ts_before'] > (3 * 60 * 1000 - 1)].index\n",
    "df_feature['ts_before'] = np.log(df_feature['ts_before'] // 1000 + 1)\n",
    "LENGTH = len(INDEX)\n",
    "ts_len = []\n",
    "group = []\n",
    "for i in tqdm(range(1, LENGTH)):\n",
    "    ts_len += [(INDEX[i] - INDEX[i - 1])] * (INDEX[i] - INDEX[i - 1])\n",
    "    group += [i] * (INDEX[i] - INDEX[i - 1])\n",
    "ts_len += [(len(df_feature) - INDEX[LENGTH - 1])] * \\\n",
    "    (len(df_feature) - INDEX[LENGTH - 1])\n",
    "group += [LENGTH] * (len(df_feature) - INDEX[LENGTH - 1])\n",
    "df_feature['ts_before_len'] = ts_len\n",
    "df_feature['group'] = group\n",
    "\n",
    "group = df_feature.groupby('deviceid')\n",
    "df_feature['ts_after'] = group['ts'].shift(-1) - group['ts'].shift(0)\n",
    "df_feature['ts_after'] = df_feature['ts_after'].fillna(3 * 60 * 1000)\n",
    "INDEX = df_feature[df_feature['ts_after'] > (3 * 60 * 1000 - 1)].index\n",
    "df_feature['ts_after'] = np.log(df_feature['ts_after'] // 1000 + 1)\n",
    "LENGTH = len(INDEX)\n",
    "ts_len = [INDEX[0]] * (INDEX[0] + 1)\n",
    "for i in tqdm(range(1, LENGTH)):\n",
    "    ts_len += [(INDEX[i] - INDEX[i - 1])] * (INDEX[i] - INDEX[i - 1])\n",
    "df_feature['ts_after_len'] = ts_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 类别交叉特征\n",
    "df_feature['devicevendor_osv'] = df_feature['device_vendor'].astype(\n",
    "    'str') + '_' + df_feature['osversion'].astype('str')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 下一次 pos\n",
    "df_feature['before_pos'] = df_feature.groupby(['deviceid'])['pos'].shift(1)\n",
    "df_feature['next_pos'] = df_feature.groupby(['deviceid'])['pos'].shift(-1)\n",
    "df_feature['diff_pos'] = df_feature['next_pos'] - df_feature['pos']\n",
    "\n",
    "# 距离变化\n",
    "df_feature['next_lat'] = df_feature.groupby(['deviceid'])['lat'].shift(-1)\n",
    "df_feature['next_lng'] = df_feature.groupby(['deviceid'])['lng'].shift(-1)\n",
    "df_feature['dist_diff'] = (df_feature['next_lat'] - df_feature['lat']\n",
    "                           ) ** 2 + (df_feature['lng'] - df_feature['next_lng']) ** 2\n",
    "\n",
    "del df_feature['next_lat']\n",
    "del df_feature['next_lng']\n",
    "\n",
    "# 下一次 网络\n",
    "df_feature['next_netmodel'] = df_feature.groupby(['deviceid'])[\n",
    "    'netmodel'].shift(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feature.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 历史特征"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## day 为单位 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对前一天的样本的所有反应时间进行统计量提取\n",
    "df_temp = df_feature[df_feature['target'] == 1]\n",
    "df_temp['click_minus'] = df_temp['timestamp'] - df_temp['ts']\n",
    "\n",
    "col = 'deviceid'\n",
    "col2 = 'click_minus'\n",
    "\n",
    "df_temp = df_temp.groupby([col, 'day'], as_index=False)[col2].agg({\n",
    "    'yesterday_{}_{}_max'.format(col, col2): 'max',\n",
    "    'yesterday_{}_{}_mean'.format(col, col2): 'mean',\n",
    "    'yesterday_{}_{}_min'.format(col, col2): 'min',\n",
    "    'yesterday_{}_{}_std'.format(col, col2): 'std',\n",
    "    'yesterday_{}_{}_median'.format(col, col2): 'median',\n",
    "    'yesterday_{}_{}_kurt'.format(col, col2): kurtosis,\n",
    "    'yesterday_{}_{}_q3'.format(col, col2): lambda x: np.quantile(x, q=0.75),\n",
    "})\n",
    "df_temp['day'] += 1\n",
    "\n",
    "df_feature = df_feature.merge(df_temp, on=[col, 'day'], how='left')\n",
    "\n",
    "del df_temp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 昨日 deviceid 点击次数，点击率\n",
    "col = 'deviceid'\n",
    "df_temp = df_feature.groupby([col, 'day'], as_index=False)['target'].agg({\n",
    "    'yesterday_{}_click_count'.format(col): 'sum',\n",
    "    'yesterday_{}_count'.format(col): 'count',\n",
    "})\n",
    "df_temp['yesterday_{}_ctr'.format(col)] = df_temp['yesterday_{}_click_count'.format(col)] \\\n",
    "    / df_temp['yesterday_{}_count'.format(col)]\n",
    "df_temp['day'] += 1\n",
    "del df_temp['yesterday_{}_count'.format(col)]\n",
    "\n",
    "df_feature = df_feature.merge(df_temp, on=[col, 'day'], how='left')\n",
    "\n",
    "del df_temp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 昨日小时点击率\n",
    "groups = ['deviceid', 'hour']\n",
    "df_temp = df_feature.groupby(groups + ['day'], as_index=False)['target'].agg({\n",
    "    'yesterday_{}_click_count'.format('_'.join(groups)): 'sum',\n",
    "    'yesterday_{}_count'.format('_'.join(groups)): 'count',\n",
    "})\n",
    "\n",
    "df_temp['yesterday_{}_ctr'.format('_'.join(groups))] = df_temp['yesterday_{}_click_count'.format('_'.join(groups))] \\\n",
    "    / df_temp['yesterday_{}_count'.format('_'.join(groups))]\n",
    "df_temp['day'] += 1\n",
    "\n",
    "del df_temp['yesterday_{}_click_count'.format('_'.join(groups))]\n",
    "del df_temp['yesterday_{}_count'.format('_'.join(groups))]\n",
    "\n",
    "df_feature = df_feature.merge(df_temp, on=groups + ['day'], how='left')\n",
    "\n",
    "del df_temp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 昨日曝光 pos 平均值\n",
    "col = 'deviceid'\n",
    "df_temp = df_feature.groupby([col, 'day'], as_index=False)['pos'].agg({\n",
    "    'yesterday_{}_pos_mean'.format(col): 'mean',\n",
    "})\n",
    "df_temp['day'] += 1\n",
    "\n",
    "df_feature = df_feature.merge(df_temp, on=[col, 'day'], how='left')\n",
    "\n",
    "del df_temp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 昨日 deviceid netmodel 点击率\n",
    "groups = ['deviceid', 'netmodel']\n",
    "df_temp = df_feature.groupby(groups + ['day'], as_index=False)['target'].agg({\n",
    "    'yesterday_{}_click_count'.format('_'.join(groups)): 'sum',\n",
    "    'yesterday_{}_count'.format('_'.join(groups)): 'count',\n",
    "})\n",
    "\n",
    "df_temp['yesterday_{}_ctr'.format('_'.join(groups))] = df_temp['yesterday_{}_click_count'.format('_'.join(groups))] \\\n",
    "    / df_temp['yesterday_{}_count'.format('_'.join(groups))]\n",
    "\n",
    "df_temp['day'] += 1\n",
    "\n",
    "df_feature = df_feature.merge(df_temp, on=groups + ['day'], how='left')\n",
    "df_feature['yesterday_deviceid_netmodel_click_ratio'] = df_feature['yesterday_deviceid_netmodel_click_count'] / \\\n",
    "    df_feature['yesterday_deviceid_click_count']\n",
    "\n",
    "del df_feature['yesterday_{}_click_count'.format('_'.join(groups))]\n",
    "del df_feature['yesterday_{}_count'.format('_'.join(groups))]\n",
    "\n",
    "del df_temp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对前一天的 newsid 所有反应时间进行统计量提取\n",
    "df_temp = df_feature[df_feature['target'] == 1]\n",
    "df_temp['click_minus'] = df_temp['timestamp'] - df_temp['ts']\n",
    "\n",
    "col = 'newsid'\n",
    "col2 = 'click_minus'\n",
    "\n",
    "df_temp = df_temp.groupby([col, 'day'], as_index=False)[col2].agg({\n",
    "    'yesterday_{}_{}_std'.format(col, col2): 'std',\n",
    "})\n",
    "df_temp['day'] += 1\n",
    "\n",
    "df_feature = df_feature.merge(df_temp, on=[col, 'day'], how='left')\n",
    "\n",
    "del df_temp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 昨日 newsid 点击次数，点击率\n",
    "col = 'newsid'\n",
    "df_temp = df_feature.groupby([col, 'day'], as_index=False)['target'].agg({\n",
    "    'yesterday_{}_click_count'.format(col): 'sum',\n",
    "    'yesterday_{}_count'.format(col): 'count',\n",
    "})\n",
    "df_temp['yesterday_{}_ctr'.format(col)] = df_temp['yesterday_{}_click_count'.format(col)] \\\n",
    "    / df_temp['yesterday_{}_count'.format(col)]\n",
    "\n",
    "df_temp['day'] += 1\n",
    "del df_temp['yesterday_{}_count'.format(col)]\n",
    "\n",
    "df_feature = df_feature.merge(df_temp, on=[col, 'day'], how='left')\n",
    "\n",
    "del df_temp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 昨日 next_pos 点击率\n",
    "col = 'next_pos'\n",
    "df_temp = df_feature.groupby([col, 'day'], as_index=False)['target'].agg({\n",
    "    'yesterday_{}_click_count'.format(col): 'sum',\n",
    "    'yesterday_{}_count'.format(col): 'count',\n",
    "})\n",
    "df_temp['yesterday_{}_ctr'.format(col)] = df_temp['yesterday_{}_click_count'.format(col)] \\\n",
    "    / df_temp['yesterday_{}_count'.format(col)]\n",
    "\n",
    "df_temp['day'] += 1\n",
    "\n",
    "del df_temp['yesterday_{}_count'.format(col)]\n",
    "del df_temp['yesterday_{}_click_count'.format(col)]\n",
    "\n",
    "df_feature = df_feature.merge(df_temp, on=[col, 'day'], how='left')\n",
    "\n",
    "del df_temp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_list = tqdm([['deviceid', 'netmodel']])\n",
    "for f1, f2 in cat_list:\n",
    "    df_feature['t_{}_count'.format(f1)] = df_feature.groupby([f1, 'day'])[\n",
    "        'id'].transform('count')\n",
    "    df_feature['t_{}_count'.format(f2)] = df_feature.groupby([f2, 'day'])[\n",
    "        'id'].transform('count')\n",
    "    df_feature['t_{}_count'.format('_'.join([f1, f2]))] = df_feature.groupby([\n",
    "        f1, f2, 'day'])['id'].transform('count')\n",
    "\n",
    "    df_feature['{}_coratio'.format('_'.join([f1, f2]))] = (df_feature['t_{}_count'.format(\n",
    "        f1)] * df_feature['t_{}_count'.format(f2)]) / df_feature['t_{}_count'.format('_'.join([f1, f2]))]\n",
    "    df_feature['yesterday_{}_coratio'.format('_'.join([f1, f2]))] = df_feature.groupby(\n",
    "        [f1, f2, 'day'])['{}_coratio'.format('_'.join([f1, f2]))].shift()\n",
    "\n",
    "    del df_feature['t_{}_count'.format(f1)]\n",
    "    del df_feature['t_{}_count'.format(f2)]\n",
    "    del df_feature['t_{}_count'.format('_'.join([f1, f2]))]\n",
    "    del df_feature['{}_coratio'.format('_'.join([f1, f2]))]\n",
    "\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feature.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 以 hour 为单位"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 一小时之前 deviceid 点击次数，点击率\n",
    "col = 'deviceid'\n",
    "df_temp = df_feature.groupby([col, 'hourl'], as_index=False)['id'].agg({\n",
    "    'pre_hour_{}_count'.format(col): 'count',\n",
    "})\n",
    "df_temp['hourl'] += 1\n",
    "\n",
    "df_feature = df_feature.merge(df_temp, on=[col, 'hourl'], how='left')\n",
    "\n",
    "del df_temp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feature.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 统计特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_list = [['deviceid'], ['guid'], ['newsid'], ['deviceid', 'pos'], ['newsid', 'pos'],\n",
    "            ['deviceid', 'guid', 'newsid'], ['deviceid', 'next_pos']]\n",
    "for f in tqdm(cat_list):\n",
    "    df_feature['{}_day_count'.format('_'.join(f))] = df_feature.groupby([\n",
    "        'day'] + f)['id'].transform('count')\n",
    "\n",
    "cat_list = [['deviceid'], ['guid'], [\n",
    "    'deviceid', 'pos'], ['deviceid', 'netmodel']]\n",
    "for f in tqdm(cat_list):\n",
    "    df_feature['{}_minute10_count'.format('_'.join(f))] = df_feature.groupby(\n",
    "        ['day', 'hour', 'minute10'] + f)['id'].transform('count')\n",
    "\n",
    "cat_list = [['deviceid', 'netmodel']]\n",
    "for f in tqdm(cat_list):\n",
    "    df_feature['{}_hour_count'.format('_'.join(f))] = df_feature.groupby([\n",
    "        'hourl'] + f)['id'].transform('count')\n",
    "\n",
    "cat_list = [['deviceid', 'group', 'pos']]\n",
    "for f in tqdm(cat_list):\n",
    "    df_feature['{}_count'.format('_'.join(f))] = df_feature.groupby(f)[\n",
    "        'id'].transform('count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "col = 'group'\n",
    "df_temp = df_feature.groupby([col], as_index=False)['ts_before'].agg({\n",
    "    '{}_ts_before_mean'.format(col): 'mean',\n",
    "    '{}_ts_before_std'.format(col): 'std'\n",
    "})\n",
    "df_feature = df_feature.merge(df_temp, on=col, how='left')\n",
    "\n",
    "del df_temp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "col = 'deviceid'\n",
    "df_temp = df_feature.groupby([col], as_index=False)['ts_after'].agg({\n",
    "    '{}_ts_after_mean'.format('deviceid'): 'mean',\n",
    "    '{}_ts_after_std'.format('deviceid'): 'std',\n",
    "    '{}_ts_after_median'.format('deviceid'): 'median',\n",
    "    '{}_ts_after_skew'.format('deviceid'): 'skew',\n",
    "})\n",
    "df_feature = df_feature.merge(df_temp, on=col, how='left')\n",
    "\n",
    "del df_temp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_temp = df_feature.groupby(['deviceid', 'hourl'], as_index=False)[\n",
    "    'target'].agg({'hour_count': 'size'})\n",
    "df_temp = df_temp.groupby(['deviceid'], as_index=False)['hour_count'].agg({\n",
    "    '{}_hour_count_mean'.format('deviceid'): 'mean'\n",
    "})\n",
    "\n",
    "df_feature = df_feature.merge(df_temp, how='left')\n",
    "\n",
    "del df_temp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feature['deviceid_hour_cumsum'] = df_feature.groupby(['deviceid', 'hourl'])[\n",
    "    'ts'].cumcount()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_temp = df_feature[['deviceid', 'day', 'deviceid_day_count']].copy(deep=True)\n",
    "df_temp.drop_duplicates(inplace=True)\n",
    "df_temp['deviceid_day_count_diff_1'] = df_temp.groupby(\n",
    "    ['deviceid'])['deviceid_day_count'].diff()\n",
    "\n",
    "del df_temp['deviceid_day_count']\n",
    "df_feature = df_feature.merge(df_temp, how='left')\n",
    "\n",
    "del df_temp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 未来一小时 deviceid, netmodel 曝光数量\n",
    "cat_list = [['deviceid', 'netmodel']]\n",
    "for f in tqdm(cat_list):\n",
    "    df_feature['temp'] = df_feature.groupby(\n",
    "        ['hourl'] + f)['id'].transform('count')\n",
    "    df_feature['next_{}_hour_count'.format('_'.join(f))] = df_feature.groupby(f)[\n",
    "        'temp'].shift(-1)\n",
    "\n",
    "    del df_feature['temp']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feature.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ts 相关特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sort_df = df_feature.sort_values('ts').reset_index(drop=True)\n",
    "for f in [['deviceid']]:\n",
    "    tmp = sort_df.groupby(f)\n",
    "    # 前x次曝光到当前的时间差\n",
    "    for gap in tqdm([2, 3, 4, 5, 8, 10, 20, 30]):\n",
    "        sort_df['{}_prev{}_exposure_ts_gap'.format(\n",
    "            '_'.join(f), gap)] = tmp['ts'].shift(0) - tmp['ts'].shift(gap)\n",
    "        tmp2 = sort_df[\n",
    "            f + ['ts', '{}_prev{}_exposure_ts_gap'.format('_'.join(f), gap)]\n",
    "        ].drop_duplicates(f + ['ts']).reset_index(drop=True)\n",
    "        df_feature = df_feature.merge(tmp2, on=f + ['ts'], how='left')\n",
    "\n",
    "del tmp2, sort_df, tmp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sort_df = df_feature.sort_values('ts').reset_index(drop=True)\n",
    "for f in [['netmodel', 'deviceid']]:\n",
    "    tmp = sort_df.groupby(f)\n",
    "    # 前x次曝光到当前的时间差\n",
    "    for gap in tqdm([2, 3]):\n",
    "        sort_df['{}_prev{}_exposure_ts_gap'.format(\n",
    "            '_'.join(f), gap)] = tmp['ts'].shift(0) - tmp['ts'].shift(gap)\n",
    "        tmp2 = sort_df[\n",
    "            f + ['ts', '{}_prev{}_exposure_ts_gap'.format('_'.join(f), gap)]\n",
    "        ].drop_duplicates(f + ['ts']).reset_index(drop=True)\n",
    "        df_feature = df_feature.merge(tmp2, on=f + ['ts'], how='left')\n",
    "\n",
    "del tmp2, sort_df, tmp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sort_df = df_feature.sort_values('ts').reset_index(drop=True)\n",
    "for f in [['deviceid']]:\n",
    "    tmp = sort_df.groupby(f)\n",
    "    # 后x次曝光到当前的时间差\n",
    "    for gap in tqdm([2, 3, 4, 5, 8, 10, 20, 30, 50]):\n",
    "        sort_df['{}_next{}_exposure_ts_gap'.format(\n",
    "            '_'.join(f), gap)] = tmp['ts'].shift(-gap) - tmp['ts'].shift(0)\n",
    "        tmp2 = sort_df[\n",
    "            f + ['ts', '{}_next{}_exposure_ts_gap'.format('_'.join(f), gap)]\n",
    "        ].drop_duplicates(f + ['ts']).reset_index(drop=True)\n",
    "        df_feature = df_feature.merge(tmp2, on=f + ['ts'], how='left')\n",
    "\n",
    "del tmp2, sort_df, tmp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sort_df = df_feature.sort_values('ts').reset_index(drop=True)\n",
    "for f in [['pos', 'deviceid']]:\n",
    "    tmp = sort_df.groupby(f)\n",
    "    # 后x次曝光到当前的时间差\n",
    "    for gap in tqdm([1, 2]):\n",
    "        sort_df['{}_next{}_exposure_ts_gap'.format(\n",
    "            '_'.join(f), gap)] = tmp['ts'].shift(-gap) - tmp['ts'].shift(0)\n",
    "        tmp2 = sort_df[\n",
    "            f + ['ts', '{}_next{}_exposure_ts_gap'.format('_'.join(f), gap)]\n",
    "        ].drop_duplicates(f + ['ts']).reset_index(drop=True)\n",
    "        df_feature = df_feature.merge(tmp2, on=f + ['ts'], how='left')\n",
    "\n",
    "del tmp2, sort_df, tmp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sort_df = df_feature.sort_values('ts').reset_index(drop=True)\n",
    "for f in [['netmodel', 'deviceid']]:\n",
    "    tmp = sort_df.groupby(f)\n",
    "    # 后x次曝光到当前的时间差\n",
    "    for gap in tqdm([1, 2]):\n",
    "        sort_df['{}_next{}_exposure_ts_gap'.format(\n",
    "            '_'.join(f), gap)] = tmp['ts'].shift(-gap) - tmp['ts'].shift(0)\n",
    "        tmp2 = sort_df[\n",
    "            f + ['ts', '{}_next{}_exposure_ts_gap'.format('_'.join(f), gap)]\n",
    "        ].drop_duplicates(f + ['ts']).reset_index(drop=True)\n",
    "        df_feature = df_feature.merge(tmp2, on=f + ['ts'], how='left')\n",
    "\n",
    "del tmp2, sort_df, tmp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sort_df = df_feature.sort_values('ts').reset_index(drop=True)\n",
    "for f in [['pos', 'netmodel', 'deviceid']]:\n",
    "    tmp = sort_df.groupby(f)\n",
    "    # 后x次曝光到当前的时间差\n",
    "    for gap in tqdm([1]):\n",
    "        sort_df['{}_next{}_exposure_ts_gap'.format(\n",
    "            '_'.join(f), gap)] = tmp['ts'].shift(-gap) - tmp['ts'].shift(0)\n",
    "        tmp2 = sort_df[\n",
    "            f + ['ts', '{}_next{}_exposure_ts_gap'.format('_'.join(f), gap)]\n",
    "        ].drop_duplicates(f + ['ts']).reset_index(drop=True)\n",
    "        df_feature = df_feature.merge(tmp2, on=f + ['ts'], how='left')\n",
    "\n",
    "del tmp2, sort_df, tmp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feature['lng_lat'] = df_feature['lng'].astype(\n",
    "    'str') + '_' + df_feature['lat'].astype('str')\n",
    "sort_df = df_feature.sort_values('ts').reset_index(drop=True)\n",
    "for f in [['deviceid', 'lng_lat']]:\n",
    "    tmp = sort_df.groupby(f)\n",
    "    # 后x次曝光到当前的时间差\n",
    "    for gap in tqdm([1]):\n",
    "        sort_df['{}_next{}_exposure_ts_gap'.format(\n",
    "            '_'.join(f), gap)] = tmp['ts'].shift(-gap) - tmp['ts'].shift(0)\n",
    "        tmp2 = sort_df[\n",
    "            f + ['ts', '{}_next{}_exposure_ts_gap'.format('_'.join(f), gap)]\n",
    "        ].drop_duplicates(f + ['ts']).reset_index(drop=True)\n",
    "        df_feature = df_feature.merge(tmp2, on=f + ['ts'], how='left')\n",
    "\n",
    "del tmp2, sort_df, tmp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sort_df = df_feature.sort_values('ts').reset_index(drop=True)\n",
    "for f in [['pos', 'deviceid', 'lng_lat']]:\n",
    "    tmp = sort_df.groupby(f)\n",
    "    # 后x次曝光到当前的时间差\n",
    "    for gap in tqdm([1]):\n",
    "        sort_df['{}_next{}_exposure_ts_gap'.format(\n",
    "            '_'.join(f), gap)] = tmp['ts'].shift(-gap) - tmp['ts'].shift(0)\n",
    "        tmp2 = sort_df[\n",
    "            f + ['ts', '{}_next{}_exposure_ts_gap'.format('_'.join(f), gap)]\n",
    "        ].drop_duplicates(f + ['ts']).reset_index(drop=True)\n",
    "        df_feature = df_feature.merge(tmp2, on=f + ['ts'], how='left')\n",
    "\n",
    "del tmp2, sort_df, tmp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for gap in tqdm([2, 3, 4, 5, 6, 7]):\n",
    "    df_feature['next_pos{}'.format(gap)] = df_feature.groupby(\n",
    "        ['deviceid'])['pos'].shift(-gap)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feature['next_pos_ts'] = df_feature['next_pos'] * \\\n",
    "    100 + df_feature['ts_after']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feature.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# user 表"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_user = pd.read_csv(os.path.join(current_path, 'raw_data', 'user.csv'))\n",
    "df_feature = df_feature.merge(\n",
    "    df_user[['deviceid', 'guid', 'level']], how='left', on=['deviceid', 'guid'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tag = df_user[['deviceid', 'tag']].copy()\n",
    "\n",
    "node_pairs = []\n",
    "for item in tqdm(df_user[['deviceid', 'tag']].values):\n",
    "    deviceid = str(item[0])\n",
    "    tags = item[1]\n",
    "\n",
    "    if type(tags) != float:\n",
    "        tags = tags.split('|')\n",
    "        for tag in tags:\n",
    "            try:\n",
    "                key, value = tag.split(':')\n",
    "            except Exception:\n",
    "                pass\n",
    "            node_pairs.append([deviceid, key, value])\n",
    "\n",
    "df_tag = pd.DataFrame(node_pairs)\n",
    "df_tag.columns = ['deviceid', 'tag', 'score']\n",
    "df_tag['score'] = df_tag['score'].astype('float')\n",
    "\n",
    "df_temp = df_tag.groupby(['deviceid'])['score'].agg({'tag_score_mean': 'mean',\n",
    "                                                     'tag_score_std': 'std',\n",
    "                                                     'tag_score_count': 'count',\n",
    "                                                     'tag_score_q2': lambda x: np.quantile(x, q=0.5),\n",
    "                                                     'tag_score_q3': lambda x: np.quantile(x, q=0.75),\n",
    "                                                     }).reset_index()\n",
    "\n",
    "df_feature = df_feature.merge(df_temp, how='left')\n",
    "\n",
    "del df_temp\n",
    "del df_tag\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import Word2Vec\n",
    "\n",
    "\n",
    "def emb(df, f1, f2):\n",
    "    emb_size = 16\n",
    "    print('====================================== {} {} ======================================'.format(f1, f2))\n",
    "    tmp = df.groupby(f1, as_index=False)[f2].agg(\n",
    "        {'{}_{}_list'.format(f1, f2): list})\n",
    "    sentences = tmp['{}_{}_list'.format(f1, f2)].values.tolist()\n",
    "    del tmp['{}_{}_list'.format(f1, f2)]\n",
    "    for i in range(len(sentences)):\n",
    "        sentences[i] = [str(x) for x in sentences[i]]\n",
    "    model = Word2Vec(sentences, size=emb_size, window=5,\n",
    "                     min_count=5, sg=0, hs=1, seed=2019)\n",
    "    emb_matrix = []\n",
    "    for seq in sentences:\n",
    "        vec = []\n",
    "        for w in seq:\n",
    "            if w in model:\n",
    "                vec.append(model[w])\n",
    "        if len(vec) > 0:\n",
    "            emb_matrix.append(np.mean(vec, axis=0))\n",
    "        else:\n",
    "            emb_matrix.append([0] * emb_size)\n",
    "\n",
    "    df_emb = pd.DataFrame(emb_matrix)\n",
    "    df_emb.columns = ['{}_{}_emb_{}'.format(\n",
    "        f1, f2, i) for i in range(emb_size)]\n",
    "\n",
    "    tmp = pd.concat([tmp, df_emb], axis=1)\n",
    "\n",
    "    del model, emb_matrix, sentences\n",
    "    return tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for f1, f2 in [['newsid', 'deviceid'], ['lng_lat', 'deviceid']]:\n",
    "    df_feature = df_feature.merge(emb(df_feature, f1, f2), on=f1, how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feature['o_d'] = df_feature['deviceid'].astype(\n",
    "    str)+'_'+df_feature['newsid'].astype(str)\n",
    "\n",
    "sentence = df_feature[['deviceid', 'newsid', 'o_d']].astype(\n",
    "    str).fillna('-1').astype(str).values\n",
    "sentence = sentence.tolist()\n",
    "print('training...')\n",
    "np.random.seed(2019)\n",
    "\n",
    "L = 5\n",
    "model = Word2Vec(sentence, size=L, window=20, min_count=3,\n",
    "                 workers=multiprocessing.cpu_count(), iter=10)\n",
    "print('outputing...')\n",
    "\n",
    "\n",
    "for fea in tqdm(['deviceid', 'newsid', 'o_d']):\n",
    "    values = df_feature[fea].unique()\n",
    "    print(len(values))\n",
    "    w2v = []\n",
    "    for i in values:\n",
    "        a = [i]\n",
    "        if str(i) in model:\n",
    "            a.extend(model[str(i)])\n",
    "        else:\n",
    "            a.extend(np.ones(L) * -10)\n",
    "        w2v.append(a)\n",
    "    w2v = pd.DataFrame(w2v)\n",
    "    w2v.columns = [fea, fea+'_w2v_1', fea+'_w2v_2', fea+'_w2v_3',\n",
    "                   fea+'_w2v_4', fea+'_w2v_5']\n",
    "    df_feature = df_feature.merge(w2v, on=fea, how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feature['o_d1'] = df_feature['lng'].astype(\n",
    "    str)+'_'+df_feature['lat'].astype(str)\n",
    "\n",
    "sentence = df_feature[['lng', 'lat', 'o_d1']].astype(\n",
    "    str).fillna('-1').astype(str).values\n",
    "sentence = sentence.tolist()\n",
    "print('training...')\n",
    "np.random.seed(2019)\n",
    "\n",
    "L = 5\n",
    "model = Word2Vec(sentence, size=L, window=20, min_count=3,\n",
    "                 workers=multiprocessing.cpu_count(), iter=10)\n",
    "print('outputing...')\n",
    "\n",
    "for fea in tqdm(['lng', 'lat', 'o_d1']):\n",
    "    values = df_feature[fea].unique()\n",
    "    print(len(values))\n",
    "    w2v = []\n",
    "    for i in values:\n",
    "        a = [i]\n",
    "        if str(i) in model:\n",
    "            a.extend(model[str(i)])\n",
    "        else:\n",
    "            a.extend(np.ones(L) * -10)\n",
    "        w2v.append(a)\n",
    "    w2v = pd.DataFrame(w2v)\n",
    "    w2v.columns = [fea, fea+'_w2v_1', fea+'_w2v_2', fea+'_w2v_3',\n",
    "                   fea+'_w2v_4', fea+'_w2v_5']\n",
    "    df_feature = df_feature.merge(w2v, on=fea, how='left')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 减少内存"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to reduce the memory usage\n",
    "def reduce_mem_usage(df, verbose=True):\n",
    "    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n",
    "    start_mem = df.memory_usage().sum() / 1024**2\n",
    "    for col in tqdm(df.columns):\n",
    "        col_type = df[col].dtypes\n",
    "        if col_type in numerics:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(\n",
    "                        np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(\n",
    "                        np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(\n",
    "                        np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(\n",
    "                        np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)\n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(\n",
    "                        np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(\n",
    "                        np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    if verbose:\n",
    "        print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(\n",
    "            end_mem, 100 * (start_mem - end_mem) / start_mem))\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feature = reduce_mem_usage(df_feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feature.to_pickle(os.path.join(current_path, 'feature', 'feature.pickle'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:dm] *",
   "language": "python",
   "name": "conda-env-dm-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}