import pandas as pd

# 分塊讀取大數(shù)據(jù)集
chunk_iter = pd.read_csv('large_dataset.csv', chunksize=100000)
for chunk in chunk_iter:
    process(chunk)  # 自定義處理函數(shù)

# 只讀取需要的列
cols = ['id', 'name', 'value']
df = pd.read_csv('data.csv', usecols=cols)

# 指定數(shù)據(jù)類型減少內(nèi)存占用
dtypes = {'id': 'int32', 'price': 'float32'}
df = pd.read_csv('data.csv', dtype=dtypes)

# 顯示所有列的統(tǒng)計(jì)信息（包括非數(shù)值列）
df.describe(include='all')

# 檢查內(nèi)存使用情況
df.info(memory_usage='deep')
# 唯一值及其計(jì)數(shù)的高級(jí)展示
for col in df.select_dtypes(include=['object']).columns:
    print(f"\n{col}的值分布:")
    print(df[col].value_counts(dropna=False).head(10))

# 可視化缺失值
import missingno as msno
msno.matrix(df)

# 基于規(guī)則填充缺失值
df['salary'] = df.groupby('department')['salary'].apply(
    lambda x: x.fillna(x.mean())
)

# 創(chuàng)建缺失值指示特征
df['age_missing'] = df['age'].isna().astype(int)

# 使用IQR方法檢測(cè)異常值
def detect_outliers(df, col):
    Q1 = df[col].quantile(0.25)
    Q3 = df[col].quantile(0.75)
    IQR = Q3 - Q1
    lower_bound = Q1 - 1.5 * IQR
    upper_bound = Q3 + 1.5 * IQR
    return ~df[col].between(lower_bound, upper_bound)

outliers = detect_outliers(df, 'price')
df['price_cleaned'] = np.where(outliers, np.nan, df['price'])

# 使用Z-score處理異常值
from scipy import stats
df['z_score'] = np.abs(stats.zscore(df['value']))
df['value_cleaned'] = np.where(df['z_score'] > 3, np.nan, df['value'])

# 使用正則表達(dá)式提取信息
df['phone_area'] = df['phone'].str.extract(r'\((\d{3})\)')

# 向量化的字符串操作
df['name'] = df['first_name'].str.cat(df['last_name'], sep=' ')

# 使用fuzzywuzzy進(jìn)行模糊匹配
from fuzzywuzzy import fuzz
df['similarity'] = df.apply(
    lambda x: fuzz.ratio(x['name1'], x['name2']), axis=1
)

# 同時(shí)計(jì)算多個(gè)聚合函數(shù)
agg_funcs = {
    'sales': ['sum', 'mean', 'max'],
    'profit': lambda x: (x > 0).mean()  # 盈利比例
}
result = df.groupby('region').agg(agg_funcs)

# 使用transform保持原始DataFrame形狀
df['dept_avg_salary'] = df.groupby('department')['salary'].transform('mean')

# 使用pivot_table進(jìn)行透視
pd.pivot_table(df, values='sales', index='region',
               columns='quarter', aggfunc=np.sum,
               margins=True, margins_name='總計(jì)')

# 基于索引的快速合并
df1.join(df2, how='left')

# 使用merge的indicator參數(shù)跟蹤合并來(lái)源
pd.merge(df1, df2, on='key', how='outer', indicator=True)

# 使用concat進(jìn)行軸向合并
pd.concat([df1, df2], axis=1, keys=['2022', '2023'])

# 重采樣與滾動(dòng)窗口
df.set_index('date').resample('W').mean()  # 按周重采樣
df.rolling(window='30D').mean()  # 30天滾動(dòng)平均

# 處理時(shí)區(qū)
df['timestamp'] = df['timestamp'].dt.tz_localize('UTC').dt.tz_convert('Asia/Shanghai')

# 時(shí)間特征工程
df['hour'] = df['timestamp'].dt.hour
df['is_weekend'] = df['timestamp'].dt.dayofweek >= 5

# 自動(dòng)優(yōu)化數(shù)據(jù)類型
def optimize_dtypes(df):
    for col in df.columns:
        col_type = df[col].dtype
        
        if col_type == 'object':
            num_unique = df[col].nunique()
            if num_unique / len(df) < 0.5:
                df[col] = df[col].astype('category')
                
        elif col_type == 'float64':
            df[col] = pd.to_numeric(df[col], downcast='float')
            
        elif col_type == 'int64':
            df[col] = pd.to_numeric(df[col], downcast='integer')
    
    return df

???????df = optimize_dtypes(df)

# 使用swifter加速apply操作
import swifter
df['new_col'] = df['text'].swifter.apply(process_text)

# 使用modin替代pandas實(shí)現(xiàn)并行處理
import modin.pandas as mpd
df = mpd.read_csv('large_file.csv')

# 各種迭代方法的性能比較
def iterrows_example(df):
    for index, row in df.iterrows():
        process(row)

def itertuples_example(df):
    for row in df.itertuples():
        process(row)

def vectorized_example(df):
    df['new_col'] = df['col1'] + df['col2']
    
# 向量化操作通常比迭代快100-1000倍

# 1. 數(shù)據(jù)加載與初步清洗
df = pd.read_csv('ecommerce.csv', parse_dates=['order_date'])
df = df[df['order_amount'] > 0]  # 過(guò)濾無(wú)效訂單

# 2. RFM分析
snapshot_date = df['order_date'].max() + pd.Timedelta(days=1)
rfm = df.groupby('customer_id').agg({
    'order_date': lambda x: (snapshot_date - x.max()).days,
    'order_id': 'count',
    'order_amount': 'sum'
})
rfm.columns = ['recency', 'frequency', 'monetary']

# 3. RFM分箱與評(píng)分
rfm['R_score'] = pd.qcut(rfm['recency'], 5, labels=[5,4,3,2,1])
rfm['F_score'] = pd.qcut(rfm['frequency'], 5, labels=[1,2,3,4,5])
rfm['M_score'] = pd.qcut(rfm['monetary'], 5, labels=[1,2,3,4,5])
rfm['RFM_score'] = rfm[['R_score','F_score','M_score']].sum(axis=1)

# 4. 客戶分層
seg_map = {
    r'[12-15]': '高價(jià)值客戶',
    r'[9-11]': '潛力客戶',
    r'[6-8]': '一般客戶',
    r'[3-5]': '流失風(fēng)險(xiǎn)客戶'
}
rfm['segment'] = rfm['RFM_score'].astype('str').replace(seg_map, regex=True)

df.eval('result = (col1 + col2) / col3', inplace=True)