{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "OSError",
     "evalue": "Cannot save file into a non-existent directory: '/mnt/d/coding/codes/ML-all/IrelandFungicideDataAnalysis/output'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[1], line 93\u001b[0m\n\u001b[1;32m     91\u001b[0m \u001b[39m# 调用清理函数和转换函数\u001b[39;00m\n\u001b[1;32m     92\u001b[0m clear()\n\u001b[0;32m---> 93\u001b[0m trans()\n",
      "Cell \u001b[0;32mIn[1], line 89\u001b[0m, in \u001b[0;36mtrans\u001b[0;34m()\u001b[0m\n\u001b[1;32m     86\u001b[0m df_data \u001b[39m=\u001b[39m pd\u001b[39m.\u001b[39mmerge(df_data1, df_data2, how\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39minner\u001b[39m\u001b[39m'\u001b[39m, on\u001b[39m=\u001b[39m[\u001b[39m'\u001b[39m\u001b[39mYear\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mCrop Type\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mCrop\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mFungicide\u001b[39m\u001b[39m'\u001b[39m])\n\u001b[1;32m     88\u001b[0m \u001b[39m# 将数据写入新的Excel文件\u001b[39;00m\n\u001b[0;32m---> 89\u001b[0m df_data\u001b[39m.\u001b[39mto_excel(output_path, index\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m)\n",
      "File \u001b[0;32m~/anaconda3/envs/pytorch20.1/lib/python3.11/site-packages/pandas/util/_decorators.py:211\u001b[0m, in \u001b[0;36mdeprecate_kwarg.<locals>._deprecate_kwarg.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    209\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    210\u001b[0m         kwargs[new_arg_name] \u001b[39m=\u001b[39m new_arg_value\n\u001b[0;32m--> 211\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n",
      "File \u001b[0;32m~/anaconda3/envs/pytorch20.1/lib/python3.11/site-packages/pandas/util/_decorators.py:211\u001b[0m, in \u001b[0;36mdeprecate_kwarg.<locals>._deprecate_kwarg.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    209\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    210\u001b[0m         kwargs[new_arg_name] \u001b[39m=\u001b[39m new_arg_value\n\u001b[0;32m--> 211\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n",
      "File \u001b[0;32m~/anaconda3/envs/pytorch20.1/lib/python3.11/site-packages/pandas/core/generic.py:2374\u001b[0m, in \u001b[0;36mNDFrame.to_excel\u001b[0;34m(self, excel_writer, sheet_name, na_rep, float_format, columns, header, index, index_label, startrow, startcol, engine, merge_cells, encoding, inf_rep, verbose, freeze_panes, storage_options)\u001b[0m\n\u001b[1;32m   2361\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mpandas\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mio\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mformats\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mexcel\u001b[39;00m \u001b[39mimport\u001b[39;00m ExcelFormatter\n\u001b[1;32m   2363\u001b[0m formatter \u001b[39m=\u001b[39m ExcelFormatter(\n\u001b[1;32m   2364\u001b[0m     df,\n\u001b[1;32m   2365\u001b[0m     na_rep\u001b[39m=\u001b[39mna_rep,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   2372\u001b[0m     inf_rep\u001b[39m=\u001b[39minf_rep,\n\u001b[1;32m   2373\u001b[0m )\n\u001b[0;32m-> 2374\u001b[0m formatter\u001b[39m.\u001b[39mwrite(\n\u001b[1;32m   2375\u001b[0m     excel_writer,\n\u001b[1;32m   2376\u001b[0m     sheet_name\u001b[39m=\u001b[39msheet_name,\n\u001b[1;32m   2377\u001b[0m     startrow\u001b[39m=\u001b[39mstartrow,\n\u001b[1;32m   2378\u001b[0m     startcol\u001b[39m=\u001b[39mstartcol,\n\u001b[1;32m   2379\u001b[0m     freeze_panes\u001b[39m=\u001b[39mfreeze_panes,\n\u001b[1;32m   2380\u001b[0m     engine\u001b[39m=\u001b[39mengine,\n\u001b[1;32m   2381\u001b[0m     storage_options\u001b[39m=\u001b[39mstorage_options,\n\u001b[1;32m   2382\u001b[0m )\n",
      "File \u001b[0;32m~/anaconda3/envs/pytorch20.1/lib/python3.11/site-packages/pandas/io/formats/excel.py:944\u001b[0m, in \u001b[0;36mExcelFormatter.write\u001b[0;34m(self, writer, sheet_name, startrow, startcol, freeze_panes, engine, storage_options)\u001b[0m\n\u001b[1;32m    940\u001b[0m     need_save \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n\u001b[1;32m    941\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    942\u001b[0m     \u001b[39m# error: Cannot instantiate abstract class 'ExcelWriter' with abstract\u001b[39;00m\n\u001b[1;32m    943\u001b[0m     \u001b[39m# attributes 'engine', 'save', 'supported_extensions' and 'write_cells'\u001b[39;00m\n\u001b[0;32m--> 944\u001b[0m     writer \u001b[39m=\u001b[39m ExcelWriter(  \u001b[39m# type: ignore[abstract]\u001b[39;00m\n\u001b[1;32m    945\u001b[0m         writer, engine\u001b[39m=\u001b[39mengine, storage_options\u001b[39m=\u001b[39mstorage_options\n\u001b[1;32m    946\u001b[0m     )\n\u001b[1;32m    947\u001b[0m     need_save \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[1;32m    949\u001b[0m \u001b[39mtry\u001b[39;00m:\n",
      "File \u001b[0;32m~/anaconda3/envs/pytorch20.1/lib/python3.11/site-packages/pandas/io/excel/_openpyxl.py:60\u001b[0m, in \u001b[0;36mOpenpyxlWriter.__init__\u001b[0;34m(self, path, engine, date_format, datetime_format, mode, storage_options, if_sheet_exists, engine_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m     56\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mopenpyxl\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mworkbook\u001b[39;00m \u001b[39mimport\u001b[39;00m Workbook\n\u001b[1;32m     58\u001b[0m engine_kwargs \u001b[39m=\u001b[39m combine_kwargs(engine_kwargs, kwargs)\n\u001b[0;32m---> 60\u001b[0m \u001b[39msuper\u001b[39m()\u001b[39m.\u001b[39m\u001b[39m__init__\u001b[39m(\n\u001b[1;32m     61\u001b[0m     path,\n\u001b[1;32m     62\u001b[0m     mode\u001b[39m=\u001b[39mmode,\n\u001b[1;32m     63\u001b[0m     storage_options\u001b[39m=\u001b[39mstorage_options,\n\u001b[1;32m     64\u001b[0m     if_sheet_exists\u001b[39m=\u001b[39mif_sheet_exists,\n\u001b[1;32m     65\u001b[0m     engine_kwargs\u001b[39m=\u001b[39mengine_kwargs,\n\u001b[1;32m     66\u001b[0m )\n\u001b[1;32m     68\u001b[0m \u001b[39m# ExcelWriter replaced \"a\" by \"r+\" to allow us to first read the excel file from\u001b[39;00m\n\u001b[1;32m     69\u001b[0m \u001b[39m# the file and later write to it\u001b[39;00m\n\u001b[1;32m     70\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mr+\u001b[39m\u001b[39m\"\u001b[39m \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_mode:  \u001b[39m# Load from existing workbook\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/pytorch20.1/lib/python3.11/site-packages/pandas/io/excel/_base.py:1313\u001b[0m, in \u001b[0;36mExcelWriter.__init__\u001b[0;34m(self, path, engine, date_format, datetime_format, mode, storage_options, if_sheet_exists, engine_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m   1309\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_handles \u001b[39m=\u001b[39m IOHandles(\n\u001b[1;32m   1310\u001b[0m     cast(IO[\u001b[39mbytes\u001b[39m], path), compression\u001b[39m=\u001b[39m{\u001b[39m\"\u001b[39m\u001b[39mcompression\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mNone\u001b[39;00m}\n\u001b[1;32m   1311\u001b[0m )\n\u001b[1;32m   1312\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39misinstance\u001b[39m(path, ExcelWriter):\n\u001b[0;32m-> 1313\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_handles \u001b[39m=\u001b[39m get_handle(\n\u001b[1;32m   1314\u001b[0m         path, mode, storage_options\u001b[39m=\u001b[39mstorage_options, is_text\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m\n\u001b[1;32m   1315\u001b[0m     )\n\u001b[1;32m   1316\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_cur_sheet \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m   1318\u001b[0m \u001b[39mif\u001b[39;00m date_format \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n",
      "File \u001b[0;32m~/anaconda3/envs/pytorch20.1/lib/python3.11/site-packages/pandas/io/common.py:734\u001b[0m, in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m    732\u001b[0m \u001b[39m# Only for write methods\u001b[39;00m\n\u001b[1;32m    733\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mr\u001b[39m\u001b[39m\"\u001b[39m \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m mode \u001b[39mand\u001b[39;00m is_path:\n\u001b[0;32m--> 734\u001b[0m     check_parent_directory(\u001b[39mstr\u001b[39m(handle))\n\u001b[1;32m    736\u001b[0m \u001b[39mif\u001b[39;00m compression:\n\u001b[1;32m    737\u001b[0m     \u001b[39mif\u001b[39;00m compression \u001b[39m!=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mzstd\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[1;32m    738\u001b[0m         \u001b[39m# compression libraries do not like an explicit text-mode\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/pytorch20.1/lib/python3.11/site-packages/pandas/io/common.py:597\u001b[0m, in \u001b[0;36mcheck_parent_directory\u001b[0;34m(path)\u001b[0m\n\u001b[1;32m    595\u001b[0m parent \u001b[39m=\u001b[39m Path(path)\u001b[39m.\u001b[39mparent\n\u001b[1;32m    596\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m parent\u001b[39m.\u001b[39mis_dir():\n\u001b[0;32m--> 597\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mOSError\u001b[39;00m(\u001b[39mrf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mCannot save file into a non-existent directory: \u001b[39m\u001b[39m'\u001b[39m\u001b[39m{\u001b[39;00mparent\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m\"\u001b[39m)\n",
      "\u001b[0;31mOSError\u001b[0m: Cannot save file into a non-existent directory: '/mnt/d/coding/codes/ML-all/IrelandFungicideDataAnalysis/output'"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "# 定义清理函数，用于清理 Excel 文件中的空格和无用列\n",
    "def clear():\n",
    "    # 遍历指定文件夹下的所有xlsx文件\n",
    "    folder_path = './data_IR'  # 请替换为你的文件夹路径\n",
    "    for filename in os.listdir(folder_path):\n",
    "        if filename.endswith(\".xlsx\"):\n",
    "            file_path = os.path.join(folder_path, filename)\n",
    "            # 使用pandas读取xlsx文件\n",
    "            df = pd.read_excel(file_path)\n",
    "\n",
    "            # 删除第一列字符串中的空格\n",
    "            df[df.columns[0]] = df[df.columns[0]].apply(lambda x: x.replace(' ', '') if isinstance(x, str) else x)\n",
    "            \n",
    "            # 删除第一行字符串中的空格\n",
    "            df.columns = df.columns.to_series().apply(lambda x: x.replace(' ', '') if isinstance(x, str) else x)\n",
    "\n",
    "            # 创建一个新的DataFrame，它只包括第一列和那些不以 \"Unnamed\" 开头的列\n",
    "            df = df[df.columns[0]].to_frame().join(df.loc[:, ~df.columns.str.startswith('Unnamed')])\n",
    "            \n",
    "            df.rename(columns={df.columns[0]: ''}, inplace=True)\n",
    "\n",
    "            # 重新保存xlsx文件，这将自动将所有列的宽度设为单列\n",
    "            df.to_excel(file_path, index=False)\n",
    "\n",
    "# 定义转换函数，用于将多个 Excel 文件转换为一个 DataFrame，并将数据写入新的 Excel 文件\n",
    "def trans():\n",
    "    # 获取当前工作目录\n",
    "    folder_path = os.path.join(os.getcwd(), 'data_IR')\n",
    "\n",
    "    # 定义要写入的新Excel文件的路径\n",
    "    output_path = os.path.join(os.getcwd(), 'output/resultIR.xlsx')\n",
    "\n",
    "    # 定义要写入的DataFrame\n",
    "    data = pd.DataFrame(columns=['Year', 'Crop Type', 'Crop', 'Fungicide', 'Hectares', 'Active Substance'])\n",
    "\n",
    "    data1 = []\n",
    "    data2 = []\n",
    "\n",
    "    # 遍历文件夹中的所有Excel文件\n",
    "    for file_name in os.listdir(folder_path):\n",
    "        if file_name.endswith('.xlsx'):\n",
    "            # 解析文件名以获取年份、作物类型和表类型\n",
    "            parts = file_name.split('_')\n",
    "            year = parts[-2]\n",
    "            crop_type = '_'.join(parts[0:-2])\n",
    "            table_type = parts[-1].split('.')[0]\n",
    "            \n",
    "            # 读取Excel文件\n",
    "            df = pd.read_excel(os.path.join(folder_path, file_name), header=0, index_col=0)\n",
    "            \n",
    "            # 将数据转换为只有三列的DataFrame\n",
    "            for fungicide in df.index:\n",
    "                for crop in df.columns:\n",
    "                    value = str(df.loc[fungicide, crop])\n",
    "\n",
    "                    if isinstance(value, str):\n",
    "                        value = value.replace(',', '').split(\".\")[0].replace('<', '0.')  # use 0.1 instead of <1\n",
    "                        try:\n",
    "                            value = float(value)\n",
    "                        except ValueError:\n",
    "                            value = 0.0\n",
    "                    else:\n",
    "                        print(value)\n",
    "                        break\n",
    "\n",
    "                    if pd.notna(value):\n",
    "                        if table_type == 'ha':\n",
    "                            #data1 = pd.concat([data, pd.DataFrame({'Year': year, 'Crop Type': crop_type, 'Crop': crop, 'Fungicide': fungicide, 'Hectares': value}, index=[0])], ignore_index=True)\n",
    "                            data1.append([year, crop_type, crop, fungicide, value])\n",
    "    \n",
    "                        elif table_type == 'ac':\n",
    "                            #data2 = pd.concat([data, pd.DataFrame({'Year': year, 'Crop Type': crop_type, 'Crop': crop, 'Fungicide': fungicide, 'Active Substance': value}, index=[0])], ignore_index=True)\n",
    "                            data2.append([year, crop_type, crop, fungicide, value])\n",
    "    \n",
    "    # 将data1和data2转换为DataFrame对象\n",
    "    df_data1 = pd.DataFrame(data1, columns=['Year', 'Crop Type', 'Crop', 'Fungicide', 'Hectares'])\n",
    " \n",
    "    \n",
    "    df_data2 = pd.DataFrame(data2, columns=['Year', 'Crop Type', 'Crop', 'Fungicide', 'Active Substance']) \n",
    " \n",
    " \n",
    "    # 使用inner方法合并df_data1和df_data2，并查找重复行\n",
    "    df_data = pd.merge(df_data1, df_data2, how='inner', on=['Year', 'Crop Type', 'Crop', 'Fungicide'])\n",
    " \n",
    "    # 将数据写入新的Excel文件\n",
    "    df_data.to_excel(output_path, index=False)\n",
    "\n",
    "# 调用清理函数和转换函数\n",
    "clear()\n",
    "trans()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Crop Type</th>\n",
       "      <th>Crop</th>\n",
       "      <th>Fungicide</th>\n",
       "      <th>Hectares</th>\n",
       "      <th>Active Substance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2000</td>\n",
       "      <td>Arable_crops</td>\n",
       "      <td>Springbarley</td>\n",
       "      <td>Azoxystrobin</td>\n",
       "      <td>2731.0</td>\n",
       "      <td>449.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2000</td>\n",
       "      <td>Arable_crops</td>\n",
       "      <td>Undersownbarley</td>\n",
       "      <td>Azoxystrobin</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2000</td>\n",
       "      <td>Arable_crops</td>\n",
       "      <td>Winterbarley</td>\n",
       "      <td>Azoxystrobin</td>\n",
       "      <td>1946.0</td>\n",
       "      <td>353.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2000</td>\n",
       "      <td>Arable_crops</td>\n",
       "      <td>Springwheat</td>\n",
       "      <td>Azoxystrobin</td>\n",
       "      <td>166.0</td>\n",
       "      <td>31.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2000</td>\n",
       "      <td>Arable_crops</td>\n",
       "      <td>Winterwheat</td>\n",
       "      <td>Azoxystrobin</td>\n",
       "      <td>3861.0</td>\n",
       "      <td>621.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Year     Crop Type             Crop     Fungicide  Hectares  \\\n",
       "0  2000  Arable_crops     Springbarley  Azoxystrobin    2731.0   \n",
       "1  2000  Arable_crops  Undersownbarley  Azoxystrobin       0.0   \n",
       "2  2000  Arable_crops     Winterbarley  Azoxystrobin    1946.0   \n",
       "3  2000  Arable_crops      Springwheat  Azoxystrobin     166.0   \n",
       "4  2000  Arable_crops      Winterwheat  Azoxystrobin    3861.0   \n",
       "\n",
       "   Active Substance  \n",
       "0             449.0  \n",
       "1               0.0  \n",
       "2             353.0  \n",
       "3              31.0  \n",
       "4             621.0  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df=pd.read_excel(\"output/resultIR.xlsx\")\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "删除前的数据数量：14109\n",
      "删除后的数据数量：12917\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "df.dropna(subset=['Fungicide'], inplace=True)\n",
    "# 统计删除前的数据数量\n",
    "print(f'删除前的数据数量：{len(df)}')\n",
    "\n",
    "\n",
    "df.loc[df['Fungicide'].str.contains('other Fungicide', case=False), 'Fungicide'] = 'Other Fungicide'\n",
    "df.loc[df['Fungicide'].str.contains('all Fungicide', case=False), 'Fungicide'] = 'all Fungicide'\n",
    "df.loc[df['Fungicide'].str.contains('otherFungicide', case=False), 'Fungicide'] = 'Other Fungicide'\n",
    "df.loc[df['Fungicide'].str.contains('allFungicide', case=False), 'Fungicide'] = 'all Fungicide'\n",
    "\n",
    "df.loc[df['Fungicide'].str.contains('allFungicide', case=False), 'Fungicide'] = 'all Fungicide'\n",
    "df.loc[df['Fungicide'].str.contains('allFungicide', case=False), 'Fungicide'] = 'all Fungicide'\n",
    "\n",
    "# 删除Fugicide列中包含'all Fungicide'和'Other Fungicide'的行\n",
    "df = df[~df['Fungicide'].str.contains('all Fungicide|Other Fungicide|allFungicide|OtherFungicide|all Fungicides|Other Fungicides|allFungicides|OtherFungicides', case=False)]\n",
    "\n",
    "# 删除Crop列中包含'Allcrops'、'Totalquantity(kg)'和'Totalarea(spha)'的行\n",
    "df = df[~df['Crop'].str.contains('Allcrops|Allcrops\\\\(spha\\\\)|Allcrops\\\\(kg\\\\)|Totalquantity\\\\(kg\\\\)|Totalarea\\\\(spha\\\\)|Totalquantity|Totalarea', case=False, regex=True)]\n",
    " \n",
    "# 统计删除后的数据数量\n",
    "print(f'删除后的数据数量：{len(df)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Arable_crops', 'Arable_Crops', 'Edible_protected_crops',\n",
       "       'Grassland_fodder_crops', 'Grassland_Fodder_crops',\n",
       "       'Outdoor_vegetable_crops', 'Soft_fruit_crops', 'Soft_Fruit_Crops',\n",
       "       'Top_fruit_crops', 'Top_Fruit_crops', 'Vegetable_crops'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Crop Type'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Arable_crops', 'Protected_crops', 'Grassland_fodder_crops',\n",
       "       'Outdoor_vegetable_crops', 'Soft_fruit', 'Top_soft_fruit'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 将名称包含Arable的值重命名为Arable_crops\n",
    "df.loc[df['Crop Type'].str.contains('Arable', case=False), 'Crop Type'] = 'Arable_crops'\n",
    "df.loc[df['Crop Type'].str.contains('Grassland|Grassl', case=False), 'Crop Type'] = 'Grassland_fodder_crops'\n",
    "df.loc[df['Crop Type'].str.contains('Protected', case=False), 'Crop Type'] = 'Protected_crops' \n",
    "df.loc[df['Crop Type'].str.contains('Soft', case=False), 'Crop Type'] = 'Soft_fruit'\n",
    "df.loc[df['Crop Type'].str.contains('Top', case=False), 'Crop Type'] = 'Top_soft_fruit'\n",
    "df.loc[df['Crop Type'].str.contains('vegetable', case=False), 'Crop Type'] = 'Outdoor_vegetable_crops'\n",
    "df['Crop Type'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 按照不同的值，将data分成不同的数据集\n",
    "data1=df[df[\"Crop Type\"]==\"Arable_crops\"].to_excel(\"temp_data_IR/Arable_crops.xlsx\") \n",
    "data2=df[df[\"Crop Type\"]==\"Protected_crops\"].to_excel(\"temp_data_IR/Protected_crops.xlsx\")\n",
    "data3=df[df[\"Crop Type\"]==\"Grassland_fodder_crops\"].to_excel(\"temp_data_IR/Grassland_fodder_crops.xlsx\")\n",
    "data4=df[df[\"Crop Type\"]==\"Outdoor_vegetable_crops\"].to_excel(\"temp_data_IR/Outdoor_vegetable_crops.xlsx\")   \n",
    "data8=df[df[\"Crop Type\"]==\"Top_soft_fruit\"].to_excel(\"temp_data_IR/Top_soft_fruit.xlsx\")\n",
    "data9=df[df[\"Crop Type\"]==\"Soft_fruit\"].to_excel(\"temp_data_IR/Soft_fruit.xlsx\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 75428 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    " \n",
    "\n",
    "# 获取不重复的数据年份和 Crop Type 信息\n",
    "years = df.iloc[:, 0].drop_duplicates().values\n",
    "crop_types = df.iloc[:, 1].drop_duplicates().values\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    " \n",
    "\n",
    "# 存储结果\n",
    "x = []\n",
    "y = []\n",
    "for year in years:\n",
    "    for crop_type in crop_types:\n",
    "        if not df[(df.iloc[:, 0] == year) & (df.iloc[:, 1] == crop_type)].empty:\n",
    "            x.append(crop_type)\n",
    "            y.append(year)\n",
    "\n",
    "# 绘制散点图\n",
    "all_combinations = [(crop_type, year) for crop_type in crop_types for year in years]\n",
    "for combination in all_combinations:\n",
    "    if combination not in zip(x, y):\n",
    "        plt.scatter(combination[0], combination[1], color='gray', alpha=0.5, s=20)\n",
    "plt.scatter(x, y, s=50)\n",
    "plt.xlabel('Crop Type')\n",
    "plt.ylabel('Year')\n",
    "plt.yticks(range(2000, 2023))\n",
    "plt.xticks(rotation=25, ha='right')\n",
    "plt.tight_layout()\n",
    "plt.savefig('output/Year_Crop Type.png', dpi=1000)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 130736 characters]",
      "text/plain": [
       "<Figure size 1000x1000 with 6 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Create output directory if it doesn't exist\n",
    "if not os.path.exists('IRoutput_plots'):\n",
    "    os.makedirs('IRoutput_plots')\n",
    "\n",
    "# Create a figure with a 3x2 grid of subplots\n",
    "fig, axs = plt.subplots(nrows=3, ncols=2, figsize=(10, 10))\n",
    "\n",
    "# Read xlsx files from directory\n",
    "for i, filename in enumerate(os.listdir('temp_data_IR')):\n",
    "    if filename.endswith('.xlsx'):\n",
    "        df = pd.read_excel(os.path.join('temp_data_IR', filename))\n",
    "\n",
    "        # Group the data by 'Year', calculate the sum of 'Hectares' and 'Active Substance'\n",
    "        grouped_data = df.groupby('Year').agg({'Hectares': 'sum', 'Active Substance': 'sum'})\n",
    "\n",
    "        # Plot data on the i-th subplot\n",
    "        row = i // 2\n",
    "        col = i % 2\n",
    "        ax = axs[row, col]\n",
    "        grouped_data.plot(kind='bar', ax=ax)\n",
    "\n",
    "        ax.set_xlabel('Year')\n",
    "        ax.set_ylabel('Sum of Hectares and Active Substance')\n",
    "\n",
    "        # Remove the file extension to use as the plot title\n",
    "        plot_title = filename.rsplit('.', 1)[0]\n",
    "        ax.set_title(plot_title)\n",
    "\n",
    "        ax.legend()\n",
    "\n",
    "# Adjust the spacing between subplots\n",
    "plt.tight_layout()\n",
    "\n",
    "# Save the plot to a file\n",
    "plot_filename = os.path.join('output_plots', 'subplot_IR.png')\n",
    "plt.savefig(plot_filename,  dpi=1000)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 52176 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Create output directory if it doesn't exist\n",
    "if not os.path.exists('IRoutput_plots'):\n",
    "    os.makedirs('IRoutput_plots')\n",
    "\n",
    "# Read xlsx files from directory\n",
    "dfs = []\n",
    "for filename in os.listdir('temp_data_IR'):\n",
    "    if filename.endswith('.xlsx'):\n",
    "        df = pd.read_excel(os.path.join('temp_data_IR', filename))\n",
    "        dfs.append(df)\n",
    "\n",
    "# Concatenate all dataframes into one\n",
    "df = pd.concat(dfs)\n",
    "\n",
    "# Group the data by 'Year' and 'Crop Type', calculate the sum of 'Hectares' and 'Active Substance'\n",
    "grouped_data = df.groupby(['Year', 'Crop Type']).agg({'Hectares': 'sum', 'Active Substance': 'sum'})\n",
    "\n",
    "# Pivot the data to get Hectares for each year and crop type\n",
    "pivoted_data = grouped_data.pivot_table(index='Year', columns='Crop Type', values='Hectares')\n",
    "# Shift the first two columns one unit to the right\n",
    "pivoted_data.iloc[:, 0:2] = pivoted_data.iloc[:, 0:2].shift(periods=1, axis=0)\n",
    "\n",
    "# Create a figure and axis object\n",
    "fig, ax = plt.subplots(figsize=(10, 6))\n",
    "\n",
    "# Define a custom color map\n",
    "colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf']\n",
    "\n",
    "# Plot data on the axis object\n",
    "pivoted_data.plot(kind='bar', ax=ax, width=2, color=colors)\n",
    "\n",
    "# Set the axis labels and title\n",
    "ax.set_xlabel('Year')\n",
    "ax.set_ylabel('Hectares')\n",
    "ax.set_title('Hectares by Crop Type and Year')\n",
    "\n",
    "# Add a legend to the plot\n",
    "ax.legend()\n",
    "\n",
    "# Save the plot to a file\n",
    "plot_filename = os.path.join('output_plots', 'hectares_IR.png')\n",
    "plt.savefig(plot_filename)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 50984 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Create output directory if it doesn't exist\n",
    "if not os.path.exists('IRoutput_plots'):\n",
    "    os.makedirs('IRoutput_plots')\n",
    "\n",
    "# Read xlsx files from directory\n",
    "dfs = []\n",
    "for filename in os.listdir('temp_data_IR'):\n",
    "    if filename.endswith('.xlsx'):\n",
    "        df = pd.read_excel(os.path.join('temp_data_IR', filename))\n",
    "        dfs.append(df)\n",
    "\n",
    "# Concatenate all dataframes into one\n",
    "df = pd.concat(dfs)\n",
    "\n",
    "# Group the data by 'Year' and 'Crop Type', calculate the sum of 'Hectares' and 'Active Substance'\n",
    "grouped_data = df.groupby(['Year', 'Crop Type']).agg({'Hectares': 'sum', 'Active Substance': 'sum'})\n",
    "\n",
    "# Pivot the data to get Hectares for each year and crop type\n",
    "pivoted_data = grouped_data.pivot_table(index='Year', columns='Crop Type', values='Active Substance')\n",
    "# Shift the first two columns one unit to the right\n",
    "pivoted_data.iloc[:, 0:2] = pivoted_data.iloc[:, 0:2].shift(periods=1, axis=0)\n",
    "\n",
    "# Create a figure and axis object\n",
    "fig, ax = plt.subplots(figsize=(10, 6))\n",
    "\n",
    "# Define a custom color map\n",
    "colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf']\n",
    "\n",
    "# Plot data on the axis object\n",
    "pivoted_data.plot(kind='bar', ax=ax, width=2, color=colors)\n",
    "\n",
    "# Set the axis labels and title\n",
    "ax.set_xlabel('Year')\n",
    "ax.set_ylabel('Hectares')\n",
    "ax.set_title('Hectares by Crop Type and Year')\n",
    "\n",
    "# Add a legend to the plot\n",
    "ax.legend()\n",
    "\n",
    "# Save the plot to a file\n",
    "plot_filename = os.path.join('output_plots', 'Active Substance_IR.png')\n",
    "plt.savefig(plot_filename)\n",
    "\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 90964 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Create output directory if it doesn't exist\n",
    "if not os.path.exists('IRoutput_plots'):\n",
    "    os.makedirs('IRoutput_plots')\n",
    "\n",
    "# Read xlsx files from directory\n",
    "dfs = []\n",
    "for filename in os.listdir('temp_data_IR'):\n",
    "    if filename.endswith('.xlsx'):\n",
    "        df = pd.read_excel(os.path.join('temp_data_IR', filename))\n",
    "        dfs.append(df)\n",
    "\n",
    "# Concatenate all dataframes into one\n",
    "df = pd.concat(dfs)\n",
    "\n",
    "# Group the data by 'Year' and 'Crop Type', calculate the sum of 'Active Substance' and 'Hectares'\n",
    "grouped_data = df.groupby(['Year', 'Crop Type']).agg({'Hectares': 'sum', 'Active Substance': 'sum'})\n",
    "\n",
    "# Calculate Active Substance per Hectare\n",
    "grouped_data['Active Substance per Hectare'] = grouped_data['Active Substance'] / grouped_data['Hectares']\n",
    "\n",
    "# Reset the index of the dataframe\n",
    "grouped_data = grouped_data.reset_index()\n",
    "\n",
    "# Create a figure and axis object\n",
    "fig, ax = plt.subplots(figsize=(10, 6))\n",
    "\n",
    "# Use seaborn to plot the data\n",
    "sns.lineplot(data=grouped_data, x='Year', y='Active Substance per Hectare', hue='Crop Type', ax=ax)\n",
    "\n",
    "# Set the axis labels and title\n",
    "ax.set_xlabel('Year')\n",
    "ax.set_ylabel('Active Substance per Hectare')\n",
    "ax.set_title('Active Substance per Hectare by Crop Type and Year')\n",
    "\n",
    "# Add a legend to the plot\n",
    "ax.legend()\n",
    "\n",
    "# Save the plot to a file\n",
    "plot_filename = os.path.join('output_plots', 'active_substance_per_hectare_IR.png')\n",
    "plt.savefig(plot_filename)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fungicide\n",
      "*Sulphur                     2002\n",
      "Ametoctradin/Dimethomorph    2012\n",
      "Ametoctradin/dimethomorph    2016\n",
      "Amisulbrom                   2020\n",
      "Azoxystrobin                 2000\n",
      "                             ... \n",
      "Tridemorph                   2000\n",
      "Trifloxystrobin              2000\n",
      "Unknownfungicide             2002\n",
      "Vinclozolin                  2002\n",
      "Zineb                        2004\n",
      "Name: Year, Length: 216, dtype: int64\n",
      "Fungicide\n",
      "*Sulphur                     2002\n",
      "Ametoctradin/Dimethomorph    2014\n",
      "Ametoctradin/dimethomorph    2020\n",
      "Amisulbrom                   2020\n",
      "Azoxystrobin                 2021\n",
      "                             ... \n",
      "Tridemorph                   2002\n",
      "Trifloxystrobin              2020\n",
      "Unknownfungicide             2021\n",
      "Vinclozolin                  2002\n",
      "Zineb                        2004\n",
      "Name: Year, Length: 216, dtype: int64\n",
      "                           First Year  Last Year  Year Range\n",
      "Fungicide                                                   \n",
      "*Sulphur                         2002       2002           0\n",
      "Fenpropidin/fenpropimorph        2000       2000           0\n",
      "Fenhexamid/tolylfluanid          2006       2006           0\n",
      "Fenamidone/mancozeb              2004       2004           0\n",
      "Vinclozolin                      2002       2002           0\n",
      "...                               ...        ...         ...\n",
      "Tebuconazole                     2000       2021          21\n",
      "Chlorothalonil                   2000       2021          21\n",
      "Fenpropimorph                    2000       2021          21\n",
      "Epoxiconazole                    2000       2021          21\n",
      "Azoxystrobin                     2000       2021          21\n",
      "\n",
      "[216 rows x 3 columns]\n",
      "['Trifloxystrobin', 'Sulphur', 'Cyprodinil', 'Mancozeb', 'Quinoxyfen', 'Tebuconazole', 'Chlorothalonil', 'Fenpropimorph', 'Epoxiconazole', 'Azoxystrobin']\n",
      "Fungicide  Azoxystrobin  Chlorothalonil  Cyprodinil  Epoxiconazole  \\\n",
      "Year                                                                 \n",
      "2000             9535.0           690.0      2587.0         3302.0   \n",
      "2002             8446.0           720.0      2703.0         6935.0   \n",
      "2003              652.0             NaN         NaN          652.0   \n",
      "2004            10744.0         10102.0      2254.0        16247.0   \n",
      "2005              106.0           249.0         NaN          509.0   \n",
      "2006             9980.0         10561.0       343.0         8822.0   \n",
      "2007              497.0           170.0         NaN            NaN   \n",
      "2008             4565.0         18505.0       965.0         9325.0   \n",
      "2009               64.0           696.0         NaN          798.0   \n",
      "2010             3209.0         16174.0      3336.0         6851.0   \n",
      "2011              490.0            61.0         NaN           19.0   \n",
      "2012             3315.0         20373.0      1728.0         4450.0   \n",
      "2013              377.0           643.0         NaN            NaN   \n",
      "2014             1806.0         21285.0      2315.0         3401.0   \n",
      "2015              390.0             2.0         NaN            NaN   \n",
      "2016             2619.0         24810.0      2055.0         7762.0   \n",
      "2017              424.0          1285.0         NaN          730.0   \n",
      "2018             2490.0         24111.0      1243.0         6260.0   \n",
      "2019               90.0             NaN         NaN            NaN   \n",
      "2020             2852.0         16812.0       653.0         7381.0   \n",
      "2021              797.0           184.0         NaN          137.0   \n",
      "\n",
      "Fungicide  Fenpropimorph  Mancozeb  Quinoxyfen  Sulphur  Tebuconazole  \\\n",
      "Year                                                                    \n",
      "2000              4198.0      18.0       962.0    309.0        3118.0   \n",
      "2002              4591.0   16683.0      1114.0    172.0        1885.0   \n",
      "2003               164.0     335.0         NaN      NaN        2413.0   \n",
      "2004              4025.0    6280.0       432.0      NaN        2818.0   \n",
      "2005                 NaN       NaN         NaN      NaN           NaN   \n",
      "2006              4260.0   13900.0       546.0     27.0        2194.0   \n",
      "2007               181.0       2.0         NaN    109.0         761.0   \n",
      "2008              5169.0    9143.0       130.0    249.0        4232.0   \n",
      "2009               416.0       NaN         NaN      NaN           NaN   \n",
      "2010              2369.0    7662.0        47.0    474.0        2322.0   \n",
      "2011               158.0      67.0         NaN      NaN         352.0   \n",
      "2012              2933.0    8376.0       313.0   1138.0        3997.0   \n",
      "2013               341.0      18.0         NaN      NaN         391.0   \n",
      "2014              3599.0    6239.0         0.0    994.0        2041.0   \n",
      "2015               109.0      54.0         NaN      1.0         182.0   \n",
      "2016              4243.0    7268.0        31.0    794.0        5008.0   \n",
      "2017               145.0       NaN         NaN      NaN          36.0   \n",
      "2018              1898.0    5058.0        52.0    597.0        3204.0   \n",
      "2019                30.0       NaN         NaN      NaN          13.0   \n",
      "2020               347.0    2505.0         2.0    151.0        3101.0   \n",
      "2021               106.0       NaN         NaN      NaN         173.0   \n",
      "\n",
      "Fungicide  Trifloxystrobin  \n",
      "Year                        \n",
      "2000                3718.0  \n",
      "2002                4087.0  \n",
      "2003                   NaN  \n",
      "2004                5536.0  \n",
      "2005                   NaN  \n",
      "2006                2385.0  \n",
      "2007                   NaN  \n",
      "2008                 415.0  \n",
      "2009                   NaN  \n",
      "2010                   NaN  \n",
      "2011                   NaN  \n",
      "2012                   NaN  \n",
      "2013                   NaN  \n",
      "2014                   NaN  \n",
      "2015                   NaN  \n",
      "2016                 252.0  \n",
      "2017                   NaN  \n",
      "2018                  57.0  \n",
      "2019                   NaN  \n",
      "2020                 715.0  \n",
      "2021                   NaN  \n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 167476 characters]",
      "text/plain": [
       "<Figure size 1500x2000 with 10 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 统计每个Fungicide值第一次出现的年份\n",
    "print(df.groupby(\"Fungicide\")[\"Year\"].min())\n",
    "\n",
    "# 统计每个Fungicide值最后一次出现的年份\n",
    "print(df.groupby(\"Fungicide\")[\"Year\"].max())\n",
    "\n",
    "# 将结果保存到csv文件中\n",
    "min_year = df.groupby(\"Fungicide\")[\"Year\"].min()\n",
    "max_year = df.groupby(\"Fungicide\")[\"Year\"].max()\n",
    "result = pd.concat([min_year, max_year], axis=1)\n",
    "result.columns = [\"First Year\", \"Last Year\"]\n",
    "result.to_excel(\"temp_data/year_range.xlsx\")\n",
    "\n",
    "# 按照杀菌剂使用的时间长短（last year - first year），进行排序\n",
    "result[\"Year Range\"] = result[\"Last Year\"] - result[\"First Year\"]\n",
    "result.sort_values(by=\"Year Range\", inplace=True)\n",
    "print(result)\n",
    "result.to_excel(\"temp_data/year_range.xlsx\")\n",
    "\n",
    "top_10 = result.tail(10)\n",
    "top_10_fungicides = top_10.index.tolist()\n",
    "print(top_10_fungicides)\n",
    "\n",
    "# 从df中提取出top_10_fungicides的数据\n",
    "top_10_df = df[df[\"Fungicide\"].isin(top_10_fungicides)] \n",
    "\n",
    "# 按照Fungicide和Year进行分组，计算Hectares的和\n",
    "top_10_df = top_10_df.groupby([\"Fungicide\", \"Year\"]).agg({\"Hectares\": \"sum\"}) \n",
    "\n",
    "# 将top_10_df中的Fungicide和Year转换为列\n",
    "top_10_df.reset_index(inplace=True) \n",
    "\n",
    "# 将top_10_df中的Fungicide的值作为列名，Hectares的值作为对应的值\n",
    "top_10_df = top_10_df.pivot(index=\"Year\", columns=\"Fungicide\", values=\"Hectares\")\n",
    "print(top_10_df)\n",
    "\n",
    "# Create a figure and axis object\n",
    "fig, axs = plt.subplots(nrows=5, ncols=2, figsize=(15, 20))\n",
    "fig.subplots_adjust(hspace=0.3)\n",
    "\n",
    "# Loop through the top 10 fungicides and plot each one on a separate axis\n",
    "for i, fungicide in enumerate(top_10_fungicides):\n",
    "    # Calculate the row and column index for the current axis\n",
    "    row = i // 2\n",
    "    col = i % 2\n",
    "\n",
    "    # Extract the data for the current fungicide\n",
    "    fungicide_df = top_10_df[fungicide]\n",
    "\n",
    "    # Plot the data on the current axis\n",
    "    axs[row, col].bar(fungicide_df.index, fungicide_df.values)\n",
    "    axs[row, col].set_xlabel('Year')\n",
    "    axs[row, col].set_ylabel('Hectares')\n",
    "    axs[row, col].set_title(f'{fungicide} Usage Over Years')\n",
    "    axs[row, col].tick_params(axis='x')\n",
    "\n",
    "    # Save the plot to a file\n",
    "    plot_filename = os.path.join('output_images', 'top10.png')\n",
    "    plt.savefig(plot_filename, dpi=1000)\n",
    "\n",
    "# Show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Arable_crops.xlsx Year\n",
      "2000    42\n",
      "2002    56\n",
      "2004    51\n",
      "2006    55\n",
      "2008    60\n",
      "2010    59\n",
      "2012    67\n",
      "2014    63\n",
      "2016    67\n",
      "2018    64\n",
      "2020    68\n",
      "Name: Fungicide, dtype: int64\n",
      "Grassland_fodder_crops.xlsx Year\n",
      "2003    12\n",
      "2005     7\n",
      "2009    16\n",
      "2013    19\n",
      "2017    22\n",
      "2021    22\n",
      "Name: Fungicide, dtype: int64\n",
      "Outdoor_vegetable_crops.xlsx Year\n",
      "2004    22\n",
      "2007    19\n",
      "2011    22\n",
      "2013    24\n",
      "2015    21\n",
      "2017    18\n",
      "2019    16\n",
      "2021    14\n",
      "Name: Fungicide, dtype: int64\n",
      "Protected_crops.xlsx Year\n",
      "2015     7\n",
      "2017    12\n",
      "2019    14\n",
      "2021    11\n",
      "Name: Fungicide, dtype: int64\n",
      "Soft_fruit.xlsx Year\n",
      "2006    19\n",
      "2010    18\n",
      "2012    21\n",
      "2014    21\n",
      "2016    11\n",
      "2018    14\n",
      "2020    11\n",
      "Name: Fungicide, dtype: int64\n",
      "Top_soft_fruit.xlsx Year\n",
      "2002    19\n",
      "2006    16\n",
      "2008    20\n",
      "2010    19\n",
      "2012    18\n",
      "2014    17\n",
      "2016    19\n",
      "2018    23\n",
      "2020    23\n",
      "Name: Fungicide, dtype: int64\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 72440 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# 创建一个空的DataFrame \n",
    "df = pd.DataFrame()\n",
    "\n",
    "# 循环读取temp_data目录下的所有xlsx文件\n",
    "for filename in os.listdir('temp_data_IR'):\n",
    "    if filename.endswith('.xlsx'):\n",
    "        # 读取文件\n",
    "        data = pd.read_excel(os.path.join('temp_data_IR', filename))\n",
    "\n",
    "        # 将年份转换为整数类型\n",
    "        data['Year'] = data['Year'].astype(int)\n",
    "\n",
    "        # 计算每年使用的杀菌剂的种类数量\n",
    "        data = data.groupby('Year')['Fungicide'].nunique()\n",
    "        print(filename, data)\n",
    "        # Create a line plot\n",
    "        data.plot(kind='line', label=filename)\n",
    "\n",
    "plt.xlabel('Year')\n",
    "plt.ylabel('Number of Unique Fungicides')\n",
    "plt.title('Relationship between Year and Fungicide Data')\n",
    "plt.legend(fontsize='small', loc='center left', bbox_to_anchor=(1, 0.5))\n",
    "# Set x-axis limits\n",
    "plt.xlim(2000, 2021)\n",
    "plt.xticks(range(2000, 2022, 2))\n",
    "plt.savefig('output_plots/relationship_between_year_and_fungicide_data.png', dpi=1000, bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                               SARIMAX Results                                \n",
      "==============================================================================\n",
      "Dep. Variable:               Hectares   No. Observations:                    8\n",
      "Model:                 ARIMA(1, 1, 1)   Log Likelihood                 -81.920\n",
      "Date:                Wed, 12 Jul 2023   AIC                            169.839\n",
      "Time:                        17:57:56   BIC                            169.677\n",
      "Sample:                             0   HQIC                           167.834\n",
      "                                  - 8                                         \n",
      "Covariance Type:                  opg                                         \n",
      "==============================================================================\n",
      "                 coef    std err          z      P>|z|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "ar.L1         -0.8916      0.758     -1.176      0.240      -2.378       0.594\n",
      "ma.L1          0.9974      1.274      0.783      0.434      -1.500       3.495\n",
      "sigma2      8.196e+08   1.68e-09   4.89e+17      0.000     8.2e+08     8.2e+08\n",
      "===================================================================================\n",
      "Ljung-Box (L1) (Q):                   1.01   Jarque-Bera (JB):                 0.82\n",
      "Prob(Q):                              0.32   Prob(JB):                         0.66\n",
      "Heteroskedasticity (H):               0.11   Skew:                             0.74\n",
      "Prob(H) (two-sided):                  0.20   Kurtosis:                         2.21\n",
      "===================================================================================\n",
      "\n",
      "Warnings:\n",
      "[1] Covariance matrix calculated using the outer product of gradients (complex-step).\n",
      "[2] Covariance matrix is singular or near-singular, with condition number 4.96e+33. Standard errors may be unstable.\n",
      "                               SARIMAX Results                                \n",
      "==============================================================================\n",
      "Dep. Variable:       Active Substance   No. Observations:                    8\n",
      "Model:                 ARIMA(1, 1, 1)   Log Likelihood                 -81.754\n",
      "Date:                Wed, 12 Jul 2023   AIC                            169.507\n",
      "Time:                        17:57:56   BIC                            169.345\n",
      "Sample:                             0   HQIC                           167.502\n",
      "                                  - 8                                         \n",
      "Covariance Type:                  opg                                         \n",
      "==============================================================================\n",
      "                 coef    std err          z      P>|z|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "ar.L1         -1.0000      0.011    -91.486      0.000      -1.021      -0.979\n",
      "ma.L1          1.0000      0.462      2.163      0.031       0.094       1.906\n",
      "sigma2      7.526e+08   7.09e-10   1.06e+18      0.000    7.53e+08    7.53e+08\n",
      "===================================================================================\n",
      "Ljung-Box (L1) (Q):                   0.56   Jarque-Bera (JB):                 3.88\n",
      "Prob(Q):                              0.45   Prob(JB):                         0.14\n",
      "Heteroskedasticity (H):               0.01   Skew:                             1.69\n",
      "Prob(H) (two-sided):                  0.01   Kurtosis:                         4.39\n",
      "===================================================================================\n",
      "\n",
      "Warnings:\n",
      "[1] Covariance matrix calculated using the outer product of gradients (complex-step).\n",
      "[2] Covariance matrix is singular or near-singular, with condition number 2.52e+33. Standard errors may be unstable.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\Anaconda\\envs\\pytorch20.1\\Lib\\site-packages\\statsmodels\\tsa\\base\\tsa_model.py:471: ValueWarning: An unsupported index was provided and will be ignored when e.g. forecasting.\n",
      "  self._init_dates(dates, freq)\n",
      "d:\\Anaconda\\envs\\pytorch20.1\\Lib\\site-packages\\statsmodels\\tsa\\base\\tsa_model.py:471: ValueWarning: An unsupported index was provided and will be ignored when e.g. forecasting.\n",
      "  self._init_dates(dates, freq)\n",
      "d:\\Anaconda\\envs\\pytorch20.1\\Lib\\site-packages\\statsmodels\\tsa\\base\\tsa_model.py:471: ValueWarning: An unsupported index was provided and will be ignored when e.g. forecasting.\n",
      "  self._init_dates(dates, freq)\n",
      "d:\\Anaconda\\envs\\pytorch20.1\\Lib\\site-packages\\statsmodels\\tsa\\statespace\\sarimax.py:978: UserWarning: Non-invertible starting MA parameters found. Using zeros as starting parameters.\n",
      "  warn('Non-invertible starting MA parameters found.'\n",
      "d:\\Anaconda\\envs\\pytorch20.1\\Lib\\site-packages\\statsmodels\\tsa\\base\\tsa_model.py:471: ValueWarning: An unsupported index was provided and will be ignored when e.g. forecasting.\n",
      "  self._init_dates(dates, freq)\n",
      "d:\\Anaconda\\envs\\pytorch20.1\\Lib\\site-packages\\statsmodels\\tsa\\base\\tsa_model.py:471: ValueWarning: An unsupported index was provided and will be ignored when e.g. forecasting.\n",
      "  self._init_dates(dates, freq)\n",
      "d:\\Anaconda\\envs\\pytorch20.1\\Lib\\site-packages\\statsmodels\\tsa\\base\\tsa_model.py:471: ValueWarning: An unsupported index was provided and will be ignored when e.g. forecasting.\n",
      "  self._init_dates(dates, freq)\n",
      "d:\\Anaconda\\envs\\pytorch20.1\\Lib\\site-packages\\statsmodels\\tsa\\statespace\\sarimax.py:978: UserWarning: Non-invertible starting MA parameters found. Using zeros as starting parameters.\n",
      "  warn('Non-invertible starting MA parameters found.'\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from statsmodels.tsa.arima.model import ARIMA\n",
    "\n",
    "# 读取数据\n",
    "df = pd.read_excel(\"D:\\\\coding\\\\codes\\\\proj2\\\\temp_data_IR\\\\Arable_crops.xlsx\")\n",
    "\n",
    "# 按年份分组，计算Hectares和Active Substance的总和\n",
    "grouped_data = df.groupby('Year').agg({'Hectares': 'sum', 'Active Substance': 'sum'})\n",
    "\n",
    "# 使用时间序列模型判断Hectares的值是否存在趋势\n",
    "model = ARIMA(grouped_data['Hectares'], order=(1, 1, 1))\n",
    "model_fit = model.fit()\n",
    "print(model_fit.summary())\n",
    "\n",
    "# 使用时间序列模型判断Active Substance的值是否存在趋势\n",
    "model = ARIMA(grouped_data['Active Substance'], order=(1, 1, 1))\n",
    "model_fit = model.fit()\n",
    "print(model_fit.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique crop types: ['Arable_crops' 'Arable_Crops' 'Edible_protected_crops'\n",
      " 'Grassland & fodder_crops' 'Grassland & Fodder_Crops'\n",
      " 'Outdoor_vegetable_crops' 'Soft_fruit_crops' 'Soft_Fruit_Crops'\n",
      " 'Top_fruit_crops' 'Top_Fruit_Crops' 'Vegetable_crops']\n",
      "F-value: 4.588524099593207\n",
      "P-value: 1.5546409090580323e-06\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import scipy.stats as stats\n",
    "df=pd.read_excel(\"output/resultIR.xlsx\")\n",
    "\n",
    "# 假设 df 是一个 DataFrame，其中 \"Crop Type\" 是作物类型， \"Hectares\" 是每公顷的杀菌剂使用量\n",
    "# 使用 pandas 的 groupby 方法，根据作物类型将数据分组\n",
    "groups = df.groupby(\"Crop Type\").groups\n",
    "unique_crop_types = df[\"Crop Type\"].unique()\n",
    " \n",
    "\n",
    "# 将每一组作物的杀菌剂使用量收集起来\n",
    "hectares_used = [df.loc[groups[name], \"Hectares\"] for name in groups]\n",
    "\n",
    "# 进行 ANOVA 分析\n",
    "f_value, p_value = stats.f_oneway(*hectares_used)\n",
    "\n",
    "print(\"F-value:\", f_value)\n",
    "print(\"P-value:\", p_value)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 22576 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 画一个神经网络模型\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "# 读取数据\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
