花书学习Day 7：Kaggle预测房价

因为需要使用Kaggle的房屋价格数据集所以先写一个download函数将数据集下载到本地import hashlib import os import tarfile import zipfile import requests DATA_HUB dict() #用于存放数据集名称映射到(数据集url,sha-1密钥) DATA_URL http://d2l-data.s3-accelerate.amazonaws.com/ def download(name, cache_diros.path.join(.., data)):# save assert name in DATA_HUB, f{name}不存在于{DATA_HUB} url, sha1_hash DATA_HUB[name] os.makedirs(cache_dir, exist_okTrue) fname os.path.join(cache_dir, url.split(/)[-1]) if os.path.exists(fname): sha1 hashlib.sha1() with open(fname, rb) as f: while True: data f.read(1048576) if not data: break sha1.update(data) if sha1.hexdigest() sha1_hash: return fname print(f正在从{url}下载{fname}...) r requests.get(url, streamTrue, verifyTrue) with open(fname, wb) as f: f.write(r.content) return fname同时实现解压缩tar/zip文件代码def download_extract(name, folderNone): #save fname download(name) base_dir os.path.dirname(fname) data_dir, ext os.path.splitext(fname) if ext .zip: fp zipfile.ZipFile(fname, r) elif ext in (.tar.gz, .gz, tgz): fp tarfile.open(fname, r) else: assert False, 只有zip/tar文件可以被解压缩 fp.extractall(base_dir) return os.path.join(base_dir, folder) if folder else data_dir def download_all(): #save for name in DATA_HUB: download(name)利用pandas的read_csv函数读取下载到的数据集并查看数据集import numpy as np import pandas as pd import torch from torch import nn DATA_HUB[kaggle_house_train] (DATA_URL kaggle_house_pred_train.csv,585e9cc93e70b39160e7921475f9bcd7d31219ce) DATA_HUB[kaggle_house_test](DATA_URL kaggle_house_pred_test.csv, fa19780a7b011d9b009e8bff8e99922a8ee2eb90) train_data pd.read_csv(download(kaggle_house_train)) test_data pd.read_csv(download(kaggle_house_test)) print(train_data.shape) print(test_data.shape)训练集总共有1460个样本80个特征和1个标签测试集右1459个样本80个特征。查看训练集样本的前四个和后两个特征以及标签有print(train_data.iloc[0:4, [0,1,2,3,-3,-2,-1]])由于数据的第一列是IDID数据能够帮助模型判断是哪个训练样本但是对于拟合数据毫无帮助因此我们去掉该数据。all_features pd.concat((train_data.iloc[:, 1:-1], test_data.iloc[:, 1:]))接下来进行数据预处理由于原始数据中包含大量空数据na、文字数据因此我们将原始数据中的空数据替换为相应特征的均值并将所有特征重新缩放到零均值和单位方差numeric_features all_features.select_dtypes(include[np.number]).columns all_features[numeric_features] all_features[numeric_features].apply(lambda x:(x - x.mean())/x.std()) all_features[numeric_features] all_features[numeric_features].fillna(0) #因为标准化后均值为0,因此将空缺值修改为0接下来处理离散值对于离散值我们可以使用独热编码all_features pd.get_dummies(all_features, dummy_naTrue, dtypefloat) print(all_features.shape)在数据预处理后数据变为330个特征将pandas数据转换为torch数据准备开始训练train_features torch.tensor(all_features[:n_train].values, dtypetorch.float32) test_features torch.tensor(all_features[n_train:].values, dtypetorch.float32) train_labels torch.tensor(train_data.iloc[:, -1].values, dtypetorch.float32) print(train_features.shape, train_labels.shape, test_features.shape)对于房价这类目标值范围很大的回归任务中计算对数均方误差。loss nn.MSELoss() in_features train_features.shape[1] def get_net(): net nn.Sequential(nn.Linear(in_features, 1)) return net def log_rmse(net, features, labels): clipped_preds torch.clamp(net(features), 1, float(inf)) rmse torch.sqrt(loss(torch.log(clipped_preds), torch.log(labels))) return rmse.item() def train(net, train_features, train_labels, test_features, test_labels, num_epochs, learning_rate, weight_decay, batch_size): train_ls, test_ls [], [] train_iter load_array((train_features, train_labels), batch_size) optimizer optim.Adam(net.parameters(), lrlearning_rate, weight_decayweight_decay) for epoch in range(num_epochs): for X, y in train_iter: optimizer.zero_grad() l loss(net(X), y) l.backward() optimizer.step() train_ls.append(log_rmse(net, train_features, train_labels)) if test_labels is not None: test_ls.append(log_rmse(net, test_features, test_labels)) return train_ls, test_ls接下来实现K折交叉验证def get_k_fold_data(k, i, X, y): assert k 1 fold_size X.shape[0] // k X_train, y_train None, None for j in range(k): idx slice(j * fold_size, (j 1) * fold_size) X_part, y_part X[idx, :], y[idx] if j i: X_valid, y_valid X_part, y_part elif X_train is None: X_train, y_train X_part, y_part else: X_train torch.cat([X_train, X_part], 0) y_train torch.cat([y_train, y_part], 0) return X_train, y_train, X_valid, y_valid def k_fold(k, X_train, y_train, num_epochs, learning_rate, weight_decay, batch_size): train_l_sum, valid_l_sum 0, 0 for i in range(k): data get_k_fold_data(k, i, X_train, y_train) net get_net() train_ls, valid_ls train(net, *data, num_epochs, learning_rate, weight_decay, batch_size) train_l_sum train_ls[-1] valid_l_sum valid_ls[-1] if i 0: plot(list(range(1, num_epochs 1)), [train_ls, valid_ls], xlabelepoch, ylabelrmse, xlim[1, num_epochs], legend[train, valid], yscalelog) print(f折{i 1}, 训练log rmse{float(train_ls[-1]):f}, f验证log rmse{float(valid_ls[-1]):f}) return train_l_sum / k, valid_l_sum / k进行训练并进行模型选择k, num_epochs, learning_rate, weight_decay, batch_size 5, 100, 5, 0, 64 train_l, valid_l k_fold(k, train_features, train_labels, num_epochs, learning_rate, weight_decay, batch_size) print(f{k}-折验证:平均训练log rmse:{float(train_l):f}, f平均验证log rmse{float(valid_l):f}) plt.show()训练并将结果保存提交Kaggledef train_and_pred(train_features,test_features, train_labels, test_data,num_epochs, lr, weigh_decay, batch_size): net get_net() train_ls, _ train(net, train_features, train_labels, None, None, num_epochs, lr, weight_decay, batch_size) plot(np.arange(1, num_epochs 1), [train_ls], xlabelepoch, ylabellog rmse,xlim[1, num_epochs],yscalelog) print(f训练log rmse:{float(train_ls[-1]):f}) preds net(test_features).detach().numpy() test_data[SalePrice] pd.Series(preds.reshape(1, -1)[0]) submission pd.concat([test_data[Id], test_data[SalePrice]], axis1) submission.to_csv(submission.csv, indexFalse) train_and_pred(train_features, test_features, train_labels, test_data,num_epochs, learning_rate, weight_decay, batch_size) plt.show()查看submission.csv将结果提交到Kaggle