eda-financial-consumer-data.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":7154155,"sourceType":"datasetVersion","datasetId":4131207}],"dockerImageVersionId":30615,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.impute import KNNImputer","metadata":{"execution":{"iopub.status.busy":"2023-12-08T15:08:02.200929Z","iopub.execute_input":"2023-12-08T15:08:02.201288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"consumer_data = pd.read_csv('/kaggle/input/financial-consumer-csv/Financial Consumer Complaints.csv')\nconsumer_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Wrangling Data","metadata":{}},{"cell_type":"code","source":"def wrangle_consumer_data(df):\n    # Drop unnecessary columns\n    df = df.drop(['Tags', 'Consumer consent provided?', 'ZIP code'], axis=1)\n    \n    # Replace spaces with underscores in column names\n    df.columns = [col.replace(' ', '_') for col in df.columns]\n    \n    # Convert date columns to datetime\n    df['Date_Sumbited'] = pd.to_datetime(df['Date_Sumbited'], format='%m/%d/%y')\n    df['Date_Received'] = pd.to_datetime(df['Date_Received'], format='%m/%d/%y')\n    \n    # Replace empty string values in 'Sub-product' and 'Sub-issue'\n    df['Sub-product'] = df['Sub-product'].replace('\"\"', \"I do not know\")\n    df['Sub-issue'] = df['Sub-issue'].replace('\"\"', \"I do not know\")\n    \n    # Fill missing values in 'Company_public_response' with a default text\n    df['Company_public_response'].fillna('Pending to provide a public response', inplace=True)\n    \n    # Find the most frequent value in the 'State' column\n    most_frequent_state = df['State'].mode()[0]\n\n    # Fill missing values with the most frequent state\n    df['State'].fillna(value=most_frequent_state, inplace=True)\n    \n    # Drop rows where 'Sub-issue' is NaN\n    df.dropna(subset=['Sub-issue'], inplace=True)\n    \n    # Replace the category values with Numerical to impute missing Values\n    dict_ = {\n    'Yes':1,\n    'No': 0\n    }\n    df['Consumer_disputed?'] = df['Consumer_disputed?'].map(dict_)\n\n    # Select the column to impute\n    column_to_impute = 'Consumer_disputed?'\n    # Create a copy of the dataset with the selected column\n    impute_data = df[[column_to_impute]].copy()\n\n    # Create an instance of the KNNImputer with the desired parameters\n    imputer = KNNImputer(n_neighbors=5)\n\n    # Fit and transform the selected column using the imputer\n    impute_data[column_to_impute] = imputer.fit_transform(impute_data[[column_to_impute]])\n\n    # Replace the original column in the dataset with the imputed values\n    df[column_to_impute] = impute_data[column_to_impute]\n\n    # Creating a dictionary for replacement\n    replacement_dict = {value: 'yes' if value >= 0.5 else 'no' for value in df['Consumer_disputed?'].unique()}\n\n    # Applying the replacement to the 'Consumer_disputed?' column\n    df['Consumer_disputed?'] = df['Consumer_disputed?'].map(replacement_dict)\n    \n    df.drop_duplicates(inplace=True)\n\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"consumer_data = wrangle_consumer_data(consumer_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"consumer_data.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"consumer_data.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the Datast to use in Power BI Dashboard\nconsumer_data.to_csv('Financial_Consumer_data.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}