panda库使用方法

3 minute read

初始化

import pandas as pd
import numpy as np
df = pd.DataFrame({'A': 'foo bar foo bar foo bar foo foo'.split(),
                   'B': 'one one two three two two one three'.split(),
                   'C': np.arange(8), 'D': np.arange(8) * 2})
print(df)


df = pd.DataFrame(index=[x for x in range(2009, 2019)])
sharp_list = []
# ……
df[stock] = sharp_list

# 超过55日高点
high_55day = max(dw.iloc[-56:-1].High.values)

索引

dw = DataFrame(columns=['Time','Open', 'High', 'Low', 'Close', 'Vol', 'ATR'], index=['Time'])

DataFrame.set_index(keys, drop=True, append=False, inplace=False, verify_integrity=False) append添加新索引,drop为False,inplace为True时,索引将会还原为列

reset_index可以还原索引,重新变为默认的整型索引 DataFrame.reset_index(level=None, drop=False, inplace=False, col_level=0, col_fill=”) level控制了具体要还原的那个等级的索引 drop为False则索引列会被还原为普通列,否则会丢失

序列化和反序列化

all_dataset.to_csv(file_name,mode='a',header=False,index=False)

dataset = DataFrame(columns=('Time','Open', 'High', 'Low', 'Close', 'Vol','RSI','slowk','slowd','fastk','fastd','macdhist','Williams %R','CCI','ULTOSC','ROC','ADX','ADXR','APO','AROONOSC','BOP','CMO','DX','MFI','MINUS_DI','MINUS_DM','MOM','PLUS_DI','PLUS_DM','PPO','TRIX','Price_Rise'))
dataset = pd.read_csv(filename,header=None,index_col=False)

#通过names可以指定读取之后的标题
df_example = pd.read_csv('Pandas_example_read.csv', names=['A', 'B','C'])

拆分

asset_dw = dw.iloc[index:index+100]
#让默认索引下标从0开始
asset_dw.reset_index(drop=True,inplace=True)
assets[symbol] = Asset.restore(quote, conf, assets_results[symbol], asset_dw)

数据类型

# 修改某一列的类型
dw['Time'] = dw['Time'].apply(str)

result = pd.DataFrame(data_list, columns=['Time', 'Open', 'High', 'Low', 'Close', 'Volume'])
# 将日期str转化成datetime类型
result['Time'] = pd.to_datetime(result['Time'], format='%Y-%m-%d')
# 将datetime类型转化成timestamp
result['Time'] = result.Time.values.astype('float64') // 10 ** 9
# 修改多列的数据类型
result = result.astype({'Open': 'float64', 'High': 'float64', 'Low': 'float64', 'Close': 'float64', 'Volume': 'float64'})
#将panda datetime转化成python datetime
current_time = row['date'].dt.to_pydatetime()

#将字符串转化成dtype datetime64[ns]
result['date'] = pd.to_datetime(result['date'], format='%Y-%m-%d')

时间序列进行合并

exchange_class = getattr(ccxt, 'binance')
ex = exchange_class({'enableRateLimit': True})
symbol = 'BTC/USDT'
k_interval = '12h'
result = ex.fetch_ohlcv(symbol, k_interval, since=None, limit=20)
dw = DataFrame(columns=('Time', 'Open', 'High', 'Low', 'Close', 'Vol'))
i = 0
for candle in result:
  dw.loc[i] = [int(candle[0]), float(candle[1]), float(candle[2]), float(candle[3]), float(candle[4]),
               float(candle[5])]
  i = i + 1
dw['Time'] = dw['Time'].apply(str)
print(dw.to_string())
freq = '24H'
dw.index = pd.to_datetime(dw['Time'],unit='ms')
high = dw['High'].resample(freq).max()
low = dw['Low'].resample(freq).min()
open = dw['Open'].resample(freq).first()
close = dw['Close'].resample(freq).last()
ts = dw['Time'].resample(freq).first()
vol = dw['Vol'].resample(freq).sum()
final_df = pd.concat([ts,open,high,low,close,vol], axis=1)
final_df.reset_index(drop=True, inplace=True)
print("after resample")
print(final_df.to_string())

查询

iloc是根据行号来查询;loc是按照索引来查询

self.dw.loc[self.dw['Time'] == ts]['Open'].iloc[0]

To select rows whose column value equals a scalar, some_value, use ==:

df.loc[df['column_name'] == some_value]

To select rows whose column value is in an iterable, some_values, use isin:

df.loc[df['column_name'].isin(some_values)]

Combine multiple conditions with &:

df.loc[(df['column_name'] >= A) & (df['column_name'] <= B)]

Note the parentheses. Due to Python’s operator precedence rules, & binds more tightly than <=and >=. Thus, the parentheses in the last example are necessary. Without the parentheses

df['column_name'] >= A & df['column_name'] <= B

is parsed as

df['column_name'] >= (A & df['column_name']) <= B

which results in a Truth value of a Series is ambiguous error.


To select rows whose column value does not equal some_value, use !=:

df.loc[df['column_name'] != some_value]

isin returns a boolean Series, so to select rows whose value is not in some_values, negate the boolean Series using ~:

df.loc[~df['column_name'].isin(some_values)]

For example,

import pandas as pd
import numpy as np
df = pd.DataFrame({'A': 'foo bar foo bar foo bar foo foo'.split(),
                   'B': 'one one two three two two one three'.split(),
                   'C': np.arange(8), 'D': np.arange(8) * 2})
print(df)
#      A      B  C   D
# 0  foo    one  0   0
# 1  bar    one  1   2
# 2  foo    two  2   4
# 3  bar  three  3   6
# 4  foo    two  4   8
# 5  bar    two  5  10
# 6  foo    one  6  12
# 7  foo  three  7  14

print(df.loc[df['A'] == 'foo'])

yields

     A      B  C   D
0  foo    one  0   0
2  foo    two  2   4
4  foo    two  4   8
6  foo    one  6  12
7  foo  three  7  14

If you have multiple values you want to include, put them in a list (or more generally, any iterable) and use isin:

print(df.loc[df['B'].isin(['one','three'])])

yields

     A      B  C   D
0  foo    one  0   0
1  bar    one  1   2
3  bar  three  3   6
6  foo    one  6  12
7  foo  three  7  14

Note, however, that if you wish to do this many times, it is more efficient to make an index first, and then use df.loc:

df = df.set_index(['B'])
print(df.loc['one'])

yields

       A  C   D
B              
one  foo  0   0
one  bar  1   2
one  foo  6  12

or, to include multiple values from the index use df.index.isin:

df.loc[df.index.isin(['one','two'])]

yields

       A  C   D
B              
one  foo  0   0
one  bar  1   2
two  foo  2   4
two  foo  4   8
two  bar  5  10
one  foo  6  12

https://thispointer.com/select-rows-columns-by-name-or-index-in-dataframe-using-loc-iloc-python-pandas/

其他

#显示所有列
pd.set_option('display.max_columns', None)
#显示所有行
pd.set_option('display.max_rows', None)
#查看列数据类型
df.dtypes
#将读取的日期转为datatime格式
df['c'] = pd.to_datetime(df['c'],format='%Y-%m-%d %H:%M:%S')
#将某个datetime类型的列转化成string
btc_df['date'] = btc_df['date'].dt.strftime('%Y-%m-%d')
#新增一个列,并且根据已有的值进行赋值,机器fgi_map是某个map,根据这个进行赋值
btc_df["fgi"] = btc_df.apply(lambda x: fgi_map[x.date] if x.date in fgi_map else np.nan, axis=1)
# column 改为 index
df.set_index('date', inplace=True) 
#(all)index 改为 column
df.reset_index()
#(the first)index 改为 column
df.reset_index(level=0, inplace=True)
# pandas的merge方法提供了一种类似于SQL的内存链接操作
result = pd.merge(left, right, on=['key1', 'key2'])