test.head()

train.isnull().sum()

train.columns

test[['0', '1', '2',
       '3', '4', '5']]=prob

# test['label']=pd.read_csv('models/v4/lgb.csv')['predict']
train=train.drop('text',axis=1)
train=train.drop('fold',axis=1)

train=train[['id', 'method', 'user_agent', 'url', 'refer', 'body', '0', '1',
       '2', '3', '4', '5', 'label']]

print("train.shape",train.shape)
print("test.shape",test.shape)

# 查看训练集的字段

train.columns

train=train.rename(columns={'lable':'label'})

train.dtypes

# 标签个数统计
train['label'].value_counts()

train['label'].value_counts().plot(kind='bar')
plt.show()

data=pd.concat([train,test],axis=0).reset_index(drop=True)
data.nunique()

# 缺失值处理：
data['user_agent']=data['user_agent'].fillna('NAN')
data['refer']=data['refer'].fillna('NAN')
data['body']=data['body'].fillna('NAN')
data['url']=data['url'].fillna('NAN')

# 提取user_agent特征
ua_cols=['browser_family', 'os_family', 'device_family','device_brand','device_model']
data[ua_cols] = data.apply(get_ua, axis=1, result_type="expand")
data.head()

import urllib.parse
import urllib

data['user_agent_len']=data['user_agent'].apply(lambda x:len(x))
data['url_len']=data['url'].apply(lambda x:len(x))
data['refer_len']=data['refer'].apply(lambda x:len(x))
data['body_len']=data['body'].apply(lambda x:len(x))
data['body_user_agent_len_diff']=data['body_len']-data['user_agent_len']
data['body_url_len_diff']=data['body_len']-data['url_len']

# 将user_agent列进行tfidf特征提取，再SVD变成16维度
texts=data['user_agent'].values.tolist()

n_components = 16      # 期望维数 
tf = TfidfVectorizer(min_df= 1, max_df=0.5,analyzer = 'char_wb', ngram_range = (1,3))     # ngram_range = (2,5)
X = tf.fit_transform(texts)
svd = TruncatedSVD(n_components=n_components,
                   random_state=42)
X_svd = svd.fit_transform(X)
df_tfidf = pd.DataFrame(X_svd)
df_tfidf.columns = [f'user_agent_name_tfidf_{i}' for i in range(n_components)]
data=pd.concat([data,df_tfidf],axis=1)

texts=data['url'].values.tolist()

n_components = 16
tf = TfidfVectorizer(min_df= 1, max_df=0.5,analyzer = 'char_wb', ngram_range = (1,3))
X = tf.fit_transform(texts)
svd = TruncatedSVD(n_components=n_components,
                   random_state=42)
X_svd = svd.fit_transform(X)
df_tfidf = pd.DataFrame(X_svd)
df_tfidf.columns = [f'url_name_tfidf_{i}' for i in range(n_components)]
data=pd.concat([data,df_tfidf],axis=1)

# texts=data['refer'].values.tolist()

# n_components = 16
# tf = TfidfVectorizer(min_df= 1, max_df=0.5,analyzer = 'char_wb', ngram_range = (1,3))
# X = tf.fit_transform(texts)
# svd = TruncatedSVD(n_components=n_components,
#                    random_state=42)
# X_svd = svd.fit_transform(X)
# df_tfidf = pd.DataFrame(X_svd)
# df_tfidf.columns = [f'refer_tfidf_{i}' for i in range(n_components)]
# data=pd.concat([data,df_tfidf],axis=1)

texts=data['body'].values.tolist()

n_components = 32
tf = TfidfVectorizer(min_df= 1, max_df=0.5,analyzer = 'char_wb', ngram_range = (1,3))
X = tf.fit_transform(texts)
svd = TruncatedSVD(n_components=n_components,
                   random_state=42)
X_svd = svd.fit_transform(X)
df_tfidf = pd.DataFrame(X_svd)
df_tfidf.columns = [f'body_tfidf_{i}' for i in range(n_components)]
data=pd.concat([data,df_tfidf],axis=1)

for f in ['method', 'url','refer', 'body','browser_family','os_family','device_family','device_brand','device_model']:    # refer
    data[f'id_{f}_nunique'] = data.groupby(['id'])[f].transform('nunique')
    data[f'id_{f}_count'] = data.groupby(['id'])[f].transform('count')

re.split('[=&]', urlparse(data['url'][0])[4])

def get_url_query(s):
    li = re.split('[=&]', urlparse(s)[4])
    return [li[i] for i in range(len(li)) if i % 2 == 1]


def find_max_str_length(x):
    max_ = 0
    li = [len(i) for i in x]
    return max(li) if len(li) > 0 else 0


def find_str_length_std(x):
    max_ = 0
    li = [len(i) for i in x]
    return np.std(li) if len(li) > 0 else -1


data['url_unquote'] = data['url'].apply(unquote)
data['url_query'] = data['url_unquote'].apply(lambda x: get_url_query(x))
data['url_query_num'] = data['url_query'].apply(len)
data['url_query_max_len'] = data['url_query'].apply(find_max_str_length)
data['url_query_len_std'] = data['url_query'].apply(find_str_length_std)
data['url'].apply(unquote)

data.head()

def find_url_filetype(x):
    try:
        return re.search(r'\.[a-z]+', x).group()
    except:
        return '__NaN__'
    
    
data['url_path'] = data['url_unquote'].apply(lambda x: urlparse(x)[2])
data['url_filetype'] = data['url_path'].apply(lambda x: find_url_filetype(x))

data['url_path_len'] = data['url_path'].apply(len)
data['url_path_num'] = data['url_path'].apply(lambda x: len(re.findall('/',  x)))

data['ua_short'] = data['user_agent'].apply(lambda x: x.split('/')[0])
data['ua_first'] = data['user_agent'].apply(lambda x: x.split(' ')[0])

# def strs_contains(strs, keyword):
#     return True if re.search(keyword, strs, re.IGNORECASE) else False

# data['url_contains_select'] = data['url_unquote'].apply(lambda x: strs_contains(x, 'select'))
# data['url_contains_select_from'] = data['url_unquote'].apply(lambda x: strs_contains(x, 'select.*from'))
# data['url_contains_union'] = data['url_unquote'].apply(lambda x: strs_contains(x, 'union'))
# data['url_contains_where'] = data['url_unquote'].apply(lambda x: strs_contains(x, 'where'))
# data['url_contains_struts2'] = data['url_unquote'].apply(lambda x: strs_contains(x, 'struts2'))
# data['url_contains_alert'] = data['url_unquote'].apply(lambda x: strs_contains(x, 'alert'))
# data['url_contains_sudo'] = data['url_unquote'].apply(lambda x: strs_contains(x, 'sudo'))
# data['url_contains_etc_passwd'] = data['url_unquote'].apply(lambda x: strs_contains(x, 'etc.*passwd'))
# data['url_contains_dot_dot'] = data['url_unquote'].apply(lambda x: strs_contains(x, '%2e%2e%2f'))
# data['url_contains_dot_dot2'] = data['url_unquote'].apply(lambda x: strs_contains(x, '\.\./'))
# data['url_contains_javascript'] = data['url_unquote'].apply(lambda x: strs_contains(x, 'javascript'))
# data['url_contains_shell'] = data['url_unquote'].apply(lambda x: strs_contains(x, 'shell'))
# data['url_contains_java_lang'] = data['url_unquote'].apply(lambda x: strs_contains(x, 'java.lang'))

for col in tqdm(['method', 'refer', 'browser_family','os_family','device_family', 'device_brand', 'device_model','url_filetype','ua_short','ua_first']):
    le = LabelEncoder()
    data[col] = le.fit_transform(data[col])

len(data.select_dtypes(include=['int','float']).columns.tolist())

col = data.select_dtypes(include=['int','float']).columns.tolist()
data = data[col]
feature_names = [i for i in col if i not in ['id','label']]

train = data[data['label'].notnull()].reset_index(drop = True)
test = data[~data['label'].notnull()].reset_index(drop = True)


x_train = train[feature_names]
y_train = train['label']
x_test = test[feature_names]

x_train.shape

def lgb_model(train, target, test, k):
    feats = [f for f in train.columns if f not in ['id','label',  'url', 'url_count']]
    #     feats=import_cols
    print('Current num of features:', len(feats))

    oof_probs = np.zeros((train.shape[0],6))
    output_preds = 0
    offline_score = []
    feature_importance_df = pd.DataFrame()
    parameters = {
        'learning_rate': 0.03,
        'boosting_type': 'gbdt',
        'objective': 'multiclass',
        'metric': 'multi_error',
        'num_class': 6,
        'num_leaves': 31,
        'feature_fraction': 0.6,
        'bagging_fraction': 0.8,
        'min_data_in_leaf': 15,
        'verbose': -1,
        'nthread': -1,
        'max_depth': 7
    }
    
    # parameters = {
    #         'learning_rate': 0.1,
    #         'metric': 'multiclass',
    #         'objective': 'multiclass',
    #         'num_classes': 6,
    #         'feature_fraction': 0.75,
    #         'bagging_fraction': 0.75,
    #         'bagging_freq': 2,
    #         'n_jobs': -1,
    #         'seed': 1029,
    #         'max_depth': 10,
    #         'num_leaves': 100,
    #         'lambda_l1': 0.5,
    #         'lambda_l2': 0.8,
    #         'verbose': -1
    #     }

    seeds = [2020]
    for seed in seeds:
        folds = StratifiedKFold(n_splits=k, shuffle=True, random_state=seed)
        for i, (train_index, test_index) in enumerate(folds.split(train, target)):
            train_y, test_y = target.iloc[train_index], target.iloc[test_index]
            train_X, test_X = train[feats].iloc[train_index, :], train[feats].iloc[test_index, :]

            dtrain = lgb.Dataset(train_X,
                                 label=train_y)
            dval = lgb.Dataset(test_X,
                               label=test_y)
            lgb_model = lgb.train(
                parameters,
                dtrain,
                num_boost_round=8000,
                valid_sets=[dval],
                # feval = evalerror,
                callbacks=[early_stopping(100), log_evaluation(100)],
            )
            oof_probs[test_index] = lgb_model.predict(test_X[feats], num_iteration=lgb_model.best_iteration) / len(
                seeds)
            offline_score.append(lgb_model.best_score['valid_0']['multi_error'])
            output_preds += lgb_model.predict(test[feats],
                                              num_iteration=lgb_model.best_iteration) / folds.n_splits / len(seeds)
            print(offline_score)
            # feature importance
            fold_importance_df = pd.DataFrame()
            fold_importance_df["feature"] = feats
            fold_importance_df["importance"] = lgb_model.feature_importance(importance_type='gain')
            fold_importance_df["fold"] = i + 1
            feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
    print('OOF-MEAN-AUC:%.6f, OOF-STD-AUC:%.6f' % (np.mean(offline_score), np.std(offline_score)))
    print('feature importance:')
    print(feature_importance_df.groupby(['feature'])['importance'].mean().sort_values(ascending=False).head(50))

    return output_preds, oof_probs, np.mean(offline_score), feature_importance_df

# feature_names = list(
#     filter(
#         lambda x: x not in ['id','label','url', 'url_count','url_query'],
#         train.columns))

print('开始模型训练train')
lgb_preds, lgb_oof, lgb_score, feature_importance_df = lgb_model(train=train[feature_names],
                                                                 target=train['label'],
                                                                 test=test[feature_names], k=5)

# xgb_params = {'n_estimators': 10000,
#               'learning_rate': 0.03689407512484644,
#               'max_depth': 8,
#               'objective': 'multi:softproba',
#               'colsample_bytree': 0.3723914688159835,
#               'subsample': 0.780714581166012,
#               'eval_metric': 'mlogloss',
#               'gamma': 0,
#               'nthread': 1,
#               'reg_lambda': 50.0,
#               'random_state': 42}

# cat_params = {'iterations': 8000,
#               'learning_rate': 0.03429054860458741,
#               'reg_lambda': 0.3242286463210283,
#               'subsample': 0.9433911589913944,
#               'random_strength': 22.4849972385133,
#               'depth': 8,
#               'thread_count': 1,
#               #         'min_data_in_leaf': 4,
#               'leaf_estimation_iterations': 8,
#               'task_type': "CPU",
#               'bootstrap_type': 'Bernoulli',
#               'verbose': 50,
#               'early_stopping_rounds': 50,
#               # 'eval_metric': 'AUC',
#               'loss_function':'MultiClass'
#               }
# # lgb = LGBMClassifier(**lgb_params)
# xgb = XGBClassifier(**xgb_params)
# cat = CatBoostClassifier(**cat_params)


# # In[18]:


# def get_oof(feats, target, test, kfold, clf):
#     oof_preds = np.zeros((feats.shape[0],6))
#     sub_preds = np.zeros((test.shape[0],6))
#     for i, (train_idx, valid_idx) in enumerate(kfold.split(feats, target)):
#         train_X, train_y = feats.loc[train_idx], target.loc[train_idx]
#         valid_X, valid_y = feats.loc[valid_idx], target.loc[valid_idx]

#         clf.fit(train_X, train_y, eval_set=[(valid_X, valid_y)], verbose=100, early_stopping_rounds=50, )
#         oof_preds[valid_idx] = clf.predict_proba(valid_X)
#         sub_preds += clf.predict_proba(test)
#         del train_X, train_y, valid_X, valid_y

#     evalution_result = accuracy_score(target, np.argmax(oof_preds,axis=1))
#     print('*' * 10)
#     print('roc auc score:', evalution_result)
#     print('*' * 20)
#     sub_preds_result = sub_preds / kfold.n_splits
#     return oof_preds, sub_preds_result


# kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=2021)
# # 45开始

# oof_preds_2, sub_preds_2 = get_oof(train[feature_names], train['label'], test[feature_names], kfold, xgb)i

# oof_preds_3, sub_preds_3 = get_oof(train[feature_names], train['label'], test[feature_names], kfold, cat)

sub['predict']=np.argmax(lgb_preds,axis=1)

sub['predict'].value_counts()

# sub.to_csv('E:/data/DF/Web攻击检测与分类识别/res/9-8-1.csv')

accuracy_score(train['label'],np.argmax(lgb_oof,axis=1))

# lgb
f1_score(np.argmax(lgb_oof,axis=1),train['label'],average= 'macro')

# xgb
# f1_score(np.argmax(oof_preds_2,axis=1),train['label'],average= 'macro')

# cat
# f1_score(np.argmax(lgb_oof,axis=1),train['label'],average= 'macro')

print(classification_report(train['label'],np.argmax(lgb_oof,axis=1)))