30 个数据工程必备的Python 包

pip install knockknock

from knockknock import email_senderfrom sklearn.linear_model import LinearRegressionimport numpy as np@email_sender(recipient_emails=["", ""], sender_email="")def train_linear_model(your_nicest_parameters):x = np.array([[1, 1], [1, 2], [2, 2], [2, 3]])y = np.dot(x, np.array([1, 2])) + 3 regression = LinearRegression().fit(x, y)return regression.score(x, y)

pip install tqdm

from tqdm import tqdMQ = 0for i in tqdm(range(10000000)):q = i +1

pip install pandas-log

import pandas as pdimport numpy as npimport pandas_logdf = pd.DataFrame({"name": ['Alfred', 'Batman', 'Catwoman'],"toy": [np.nan, 'Batmobile', 'Bullwhip'],"born": [pd.NaT, pd.Timestamp("1940-04-25"), pd.NaT]})

with pandas_log.enable():res = (df.drop("born", axis = 1).groupby('name'))

pip install emoji

import emojiprint(emoji.emojize('Python is :thumbs_up:'))

pip install thefuzz

from thefuzz import fuzz, process#Testing the score between two sentencesfuzz.ratio("Test the Word", "test the Word!")

choices = ["Atlanta Falcons", "New York Jets", "New York Giants", "Dallas Cowboys"]process.extract("new york jets", choices, limit=2)

pip install numerizer

from numerizer import numerizenumerize('forty two')

numerize('forty-two')

numerize('nine and three quarters')

numerize('maybe around nine and three quarters')

pip install pyautogui

import pyautoguipyautogui.moveTo(10, 15)pyautogui.click()pyautogui.doubleClick()pyautogui.press('enter')

pip install weightedcalcs

import seaborn as snsdf = sns.load_dataset('mpg')import weightedcalcs as wccalc = wc.Calculator("mpg")

calc.distribution(df, "origin")

pip install scikit-posthocs

import statsmodels.api as saimport statsmodels.fORMula.api as sfaimport scikit_posthocs as spdf = sa.datasets.get_rdataset('iris').datadf.columns = df.columns.str.replace('.', '')lm = sfa.ols('SepalWidth ~ C(Species)', data=df).fit()anova = sa.stats.anova_lm(lm)print(anova)

sp.posthoc_ttest(df, val_col='SepalWidth', group_col='Species', p_adjust='holm')

pip install cerberus

from cerberus import Validatorschema = {'name': {'type': 'string'}, 'gender':{'type': 'string'}, 'age':{'type':'integer'}}v = Validator(schema)

document = {'name': 'john doe', 'gender':'male', 'age': 15}v.validate(document)

pip install ppscore

import seaborn as snsimport ppscore as ppsdf = sns.load_dataset('mpg')pps.predictors(df, 'mpg')

pip install maya

import mayanow = maya.now()print(now)

tomorrow = maya.when('tomorrow')tomorrow.datetime()

pip install pendulum

import pendulumnow = pendulum.now("Europe/Berlin")now.in_timezone("Asia/Tokyo")now.to_iso8601_string()now.add(days=2)

pip install category_encoders

from category_encoders import BinaryEncoderimport pandas as pdenc = BinaryEncoder(cols=['origin']).fit(df)numeric_dataset = enc.transform(df)numeric_dataset.head()

pip install scikit-multilearn

from skmultilearn.dataset import load_datasetfrom skmultilearn.adapt import MLkNNimport sklearn.metrics as metricsX_train, y_train, feature_names, label_names = load_dataset('emotions', 'train')X_test, y_test, _, _ = load_dataset('emotions', 'test')classifier = MLkNN(k=3)prediction = classifier.fit(X_train, y_train).predict(X_test)metrics.hamming_loss(y_test, prediction)

pip install multiset

from multiset import Multisetset1 = Multiset('aab')set1

pip install jazzit

from jazzit import error_track@error_track("curb_your_enthusiasm.mp3", wait=5)def run():for num in reversed(range(10)):print(10/num)

pip install handcalcs

import handcalcs.renderfrom math import sqrt

%%rendera = 4b = 6c = sqrt(3*a + b/7)

pip install neattext

import neattext as nt mytext = "This is the word sample but ,our WEBSITE is https://exaempleeele.com ✨."docx = nt.TextFrame(text=mytext)

docx.describe()

docx.normalize()

pip install combo

from sklearn.tree import DecisionTreeClassifierfrom sklearn.linear_model import LoGISticRegressionfrom sklearn.ensemble import GradientBoostinGClassifierfrom sklearn.ensemble import RandomForestClassifierfrom sklearn.neighbors import KNeighborsClassifierfrom sklearn.model_selection import train_test_splitfrom sklearn.datasets import load_breast_cancerfrom combo.models.classifier_stacking import Stackingfrom combo.utils.data import evaluate_print

# Define data file and read X and yrandom_state = 42X, y = load_breast_cancer(return_X_y=True)X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4,random_state=random_state)# initialize a group of clfsclassifiers = [DecisionTreeClassifier(random_state=random_state),LogisticRegression(random_state=random_state),KNeighborsClassifier(),RandomForestClassifier(random_state=random_state),GradientBoostingClassifier(random_state=random_state)]clf_names = ['DT', 'LR', 'KNN', 'RF', 'GBDT']for i, clf in enumerate(classifiers):clf.fit(X_train, y_train)y_test_predict = clf.predict(X_test)evaluate_print(clf_names[i] + ' | ', y_test, y_test_predict)print()

clf = Stacking(classifiers, n_folds=4, shuffle_data=False,keep_original=True, use_proba=False,random_state=random_state)clf.fit(X_train, y_train)y_test_predict = clf.predict(X_test)evaluate_print('Stacking | ', y_test, y_test_predict)

pip install pyaztro

import pyaztropyaztro.Aztro(sign='gemini').description

pip install Faker

from faker import Fakerfake = Faker()

fake.name()

pip install fairlearn

from fairlearn.metrics import MetricFrame, selection_ratefrom fairlearn.datasets import fetch_adultdata = fetch_adult(as_frame=True)X = data.datay_true = (data.target == '>50K') * 1sex = X['sex']selection_rates = MetricFrame(metrics=selection_rate,y_true=y_true,y_pred=y_true,sensitive_features=sex)fig = selection_rates.by_group.plot.bar(legend=False, rot=0,title='Fraction earning over $50,000')

pip install tiobeindexpy

from tiobeindexpy import tiobeindexpy as tbdf = tb.top_20()

pip install pytrends

from pytrends.request import TrendReqimport pandas as pdpytrend = TrendReq()keywords = pytrend.suggestions(keyword='Present Gift')df = pd.DataFrame(keywords)df

pip install visions

import seaborn as snsfrom visions.functional import detect_type, infer_typefrom visions.typesets import CompleteSetdf = sns.load_dataset('titanic')typeset = CompleteSet()converting everything to stringsprint(detect_type(df, typeset))

pip install schedule

import scheduleimport timedef job():print("I'm working...")schedule.every(10).seconds.do(job)while True:schedule.run_pending()time.sleep(1)

pip install autocorrect

from autocorrect import Spellerspell = Speller()spell("I'm not sleaspy and tehre is no place I'm giong to.")

pip install funcy

from funcy import select, evenselect(even, {i for i in range (20)})

pip install icecream

from icecream import icdef some_function(i):i = 4 + (1 * 2)/ 10 return i + 35ic(some_function(121))

def foo():ic()if some_function(12):ic()else:ic()foo()