From cb065f907da19490fb2c59985cc48def996e234b Mon Sep 17 00:00:00 2001
From: Maciej Wielgosz <maciej.wielgosz@nibio.no>
Date: Tue, 2 Jan 2024 14:46:15 +0100
Subject: [PATCH] first version of the simple model implemented
---
.gitignore | 2 +
README.md | 67 ++++++++
notebooks/train_model.ipynb | 207 +++++++++++++++++++++++
pipeline/data_loader.py | 2 +
prepare_data/__init__.py | 0
prepare_data/clean_file_names.py | 39 +++++
prepare_data/preparare_train_val_test.py | 143 ++++++++++++++++
7 files changed, 460 insertions(+)
create mode 100644 .gitignore
create mode 100644 notebooks/train_model.ipynb
create mode 100644 pipeline/data_loader.py
create mode 100644 prepare_data/__init__.py
create mode 100644 prepare_data/clean_file_names.py
create mode 100644 prepare_data/preparare_train_val_test.py
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..f1955c4
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+/simple-needles-2-class
+/data
diff --git a/README.md b/README.md
index 65bd051..e00c2b2 100644
--- a/README.md
+++ b/README.md
@@ -4,6 +4,73 @@
The original dataset is from [here](https://zenodo.org/records/4446842).
+The local NIBIO copy of the dataset is [here](https://nibio-my.sharepoint.com/:f:/g/personal/maciej_wielgosz_nibio_no/Elq2J4unUjFOtVKnFwDvgecBnTRLdBm_F5H__1V1gs_upQ?e=XFoOl5)
+
+
+# Austrian Tree Dataset Overview
+
+Collected by: **Austrian Federal Forests AG**
+Collection Period: **Autumn 2009 - Spring 2010**
+Usage: **Non-commercial research only**
+
+## Publication Reference
+Fiel, S. & Sablatnig, R. (2010): Leaf classification using local features. In: Proc. of 34th annual Workshop of the Austrian Association for Pattern Recognition (AAPR), 2010, 69-74 pdf.
+
+## Dataset Contents
+
+### 1. Leaves of Broad Leaf Trees
+- **Total Images:** 134
+- **Types:**
+ - Ash (25 images)
+ - Beech (30 images)
+ - Hornbeam (34 images)
+ - Mountain Oak (22 images)
+ - Sycamore Maple (23 images)
+- **Details:**
+ - Image Scale: 800 pixels height or 600 pixels width
+ - Note: Ash leaves are compound, specifically pinnate
+
+### 2. Bark of Trees
+- **Total Images:** 1183
+- **Types:**
+ - Ash (34 images)
+ - Beech (16 images)
+ - Black Pine (166 images, divided into 3 age-based sub-classes)
+ - Fir (127 images, divided into 3 age-based sub-classes)
+ - Hornbeam (42 images)
+ - Larch (200 images, divided into 3 age-based sub-classes)
+ - Mountain Oak (77 images)
+ - Scots Pine (190 images, divided into 3 age-based sub-classes)
+ - Spruce (213 images, divided into 3 age-based sub-classes)
+ - Swiss Stone Pine (96 images)
+ - Sycamore Maple (22 images)
+- **Details:**
+ - Image Scale: 800 pixels height or 600 pixels width
+ - Age Categories:
+ - Less than 60 years
+ - 60 to 80 years
+ - More than 80 years
+
+### 3. Needles of Conifers
+- **Total Images:** 275
+- **Types:**
+ - Black Pine (107 images)
+ - Fir (10 images)
+ - Larch (114 images)
+ - Scots Pine (10 images)
+ - Spruce (13 images)
+ - Swiss Stone Pine (21 images)
+- **Details:**
+ - Needle Classes:
+ - Separate growth (Fir, Spruce)
+ - Cluster growth (others)
+ - Lighting:
+ - Perfect conditions (Fir, Scots Pine, Spruce)
+ - Natural conditions (others)
+
+
+
+
## Getting started
diff --git a/notebooks/train_model.ipynb b/notebooks/train_model.ipynb
new file mode 100644
index 0000000..f4ccb18
--- /dev/null
+++ b/notebooks/train_model.ipynb
@@ -0,0 +1,207 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from torchvision import transforms, datasets\n",
+ "from torch.utils.data import DataLoader\n",
+ "\n",
+ "# Define a transform to apply to each image\n",
+ "transform = transforms.Compose([\n",
+ " transforms.Resize((256, 256)), # Resize each image to 256x256\n",
+ " transforms.ToTensor(), # Convert image to a PyTorch tensor\n",
+ " transforms.Normalize(mean=[0.485, 0.456, 0.406], # Normalize for pre-trained models\n",
+ " std=[0.229, 0.224, 0.225])\n",
+ "])\n",
+ "\n",
+ "data_path = \"/home/nibio/mutable-outside-world/code/ml-department-workshop/data\"\n",
+ "\n",
+ "\n",
+ "# Create a dataset for each set: train, validation, and test\n",
+ "train_dataset = datasets.ImageFolder(root=data_path + '/train', transform=transform)\n",
+ "val_dataset = datasets.ImageFolder(root=data_path + '/val', transform=transform)\n",
+ "test_dataset = datasets.ImageFolder(root=data_path + '/test', transform=transform)\n",
+ "\n",
+ "# Create a DataLoader for each set\n",
+ "train_loader = DataLoader(train_dataset, batch_size=4, shuffle=True)\n",
+ "val_loader = DataLoader(val_dataset, batch_size=4, shuffle=False)\n",
+ "test_loader = DataLoader(test_dataset, batch_size=4, shuffle=False)\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "import torchvision\n",
+ "from torchvision import transforms, datasets\n",
+ "from torch.utils.data import DataLoader\n",
+ "\n",
+ "# Assuming 'train_dataset' is already defined and loaded as in previous examples\n",
+ "# Make sure you have the 'train_dataset.class_to_idx' attribute available\n",
+ "# which is automatically created when using datasets.ImageFolder\n",
+ "\n",
+ "# Function to show an image with labels\n",
+ "def imshow(img, labels):\n",
+ " img = img / 2 + 0.5 # unnormalize\n",
+ " npimg = img.numpy()\n",
+ " plt.imshow(np.transpose(npimg, (1, 2, 0)))\n",
+ " # Display labels below the image\n",
+ " plt.xticks([]) # Remove x-axis ticks\n",
+ " plt.yticks([]) # Remove y-axis ticks\n",
+ " plt.xlabel(' - '.join('%5s' % train_dataset.classes[label] for label in labels), fontsize=10)\n",
+ " plt.show()\n",
+ "\n",
+ "# Define transformations\n",
+ "transform = transforms.Compose([\n",
+ " transforms.Resize((256, 256)),\n",
+ " transforms.ToTensor(),\n",
+ " transforms.Normalize((0.5,), (0.5,))\n",
+ "])\n",
+ "\n",
+ "# Create the train_dataset and train_loader as before\n",
+ "train_dataset = datasets.ImageFolder(root=data_path + '/train', transform=transform)\n",
+ "train_loader = DataLoader(train_dataset, batch_size=4, shuffle=True)\n",
+ "\n",
+ "# Get some random training images\n",
+ "dataiter = iter(train_loader)\n",
+ "images, labels = next(dataiter)\n",
+ "\n",
+ "# Show images with labels\n",
+ "imshow(torchvision.utils.make_grid(images), labels)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# create a simple CNN model\n",
+ "import torch\n",
+ "import torch.nn as nn\n",
+ "import torch.nn.functional as F\n",
+ "import torch.optim as optim\n",
+ "\n",
+ "\n",
+ "class SimpleCNN(nn.Module):\n",
+ " def __init__(self):\n",
+ " super(SimpleCNN, self).__init__()\n",
+ " self.conv1 = nn.Conv2d(in_channels=3, out_channels=16, kernel_size=3, padding=1)\n",
+ " self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n",
+ " self.conv2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=3, padding=1)\n",
+ " self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n",
+ " self.conv3 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, padding=1)\n",
+ " self.pool3 = nn.MaxPool2d(kernel_size=2, stride=2)\n",
+ " self.fc1 = nn.Linear(in_features=64 * 32 * 32, out_features=500)\n",
+ " self.fc2 = nn.Linear(in_features=500, out_features=2)\n",
+ "\n",
+ " def forward(self, x):\n",
+ " x = self.pool1(F.relu(self.conv1(x))) # 16 x 128 x 128\n",
+ " x = self.pool2(F.relu(self.conv2(x))) # 32 x 64 x 64\n",
+ " x = self.pool3(F.relu(self.conv3(x))) # 64 x 32 x 32\n",
+ " x = x.view(-1, 64 * 32 * 32) # Flatten\n",
+ " x = F.relu(self.fc1(x))\n",
+ " x = self.fc2(x)\n",
+ " return x\n",
+ " \n",
+ "# train the model\n",
+ "\n",
+ "\n",
+ "# Create an instance of the model\n",
+ "model = SimpleCNN()\n",
+ "\n",
+ "# Define the loss function and optimizer\n",
+ "criterion = nn.CrossEntropyLoss()\n",
+ "\n",
+ "# Use Adam optimizer\n",
+ "optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
+ "\n",
+ "# Train the model\n",
+ "num_epochs = 5\n",
+ "for epoch in range(num_epochs):\n",
+ " running_loss = 0.0\n",
+ " for i, data in enumerate(train_loader):\n",
+ " # Get the inputs\n",
+ " inputs, labels = data\n",
+ "\n",
+ " # Zero the parameter gradients\n",
+ " optimizer.zero_grad()\n",
+ "\n",
+ " # Forward + backward + optimize\n",
+ " outputs = model(inputs)\n",
+ " loss = criterion(outputs, labels)\n",
+ " loss.backward()\n",
+ " optimizer.step()\n",
+ "\n",
+ " # Print statistics\n",
+ " print('[%d, %5d] loss: %.3f' % (epoch + 1, i + 1, loss))\n",
+ "\n",
+ "\n",
+ "# save the model\n",
+ "torch.save(model.state_dict(), 'simple_cnn.pth')\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Accuracy of the network on the test images: 77 %\n"
+ ]
+ }
+ ],
+ "source": [
+ "# load the model\n",
+ "model = SimpleCNN()\n",
+ "model.load_state_dict(torch.load('simple_cnn.pth'))\n",
+ "\n",
+ "# run the model on the test set and print the accuracy\n",
+ "correct = 0\n",
+ "total = 0\n",
+ "\n",
+ "with torch.no_grad():\n",
+ " for data in test_loader:\n",
+ " images, labels = data\n",
+ " outputs = model(images)\n",
+ " _, predicted = torch.max(outputs.data, dim=1)\n",
+ " total += labels.size(0)\n",
+ " correct += (predicted == labels).sum().item()\n",
+ "\n",
+ "print('Accuracy of the network on the test images: %d %%' % (100 * correct / total))"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/pipeline/data_loader.py b/pipeline/data_loader.py
new file mode 100644
index 0000000..5bc70a2
--- /dev/null
+++ b/pipeline/data_loader.py
@@ -0,0 +1,2 @@
+import torch
+# import dataset from torch
diff --git a/prepare_data/__init__.py b/prepare_data/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/prepare_data/clean_file_names.py b/prepare_data/clean_file_names.py
new file mode 100644
index 0000000..4209668
--- /dev/null
+++ b/prepare_data/clean_file_names.py
@@ -0,0 +1,39 @@
+import sys
+import os
+import re
+
+
+def clean_file_names(path):
+ """
+ Clean file names in a directory. This function will replace all spaces with underscores,
+ replace all dashes with underscores, and change all file names to lowercase. If there are
+ numbers in brackets, they will be replaced with an underscore and the number.
+
+ Parameters
+ ----------
+ path : str
+ Path to directory containing files to be renamed.
+
+ Returns
+ -------
+ None.
+
+ """
+
+ for filename in os.listdir(path):
+ if filename.lower().endswith((".png", ".jpg")):
+ # replace all spaces with underscores
+ new_filename = re.sub(r"\s+", "_", filename)
+ # replace all dashes with underscores
+ new_filename = re.sub(r"-", "_", new_filename)
+ # if there are numbers in bruckets, change to underscore number
+ new_filename = re.sub(r"\(\d+\)", lambda x: "_" + x.group()[1:-1], new_filename)
+ print(new_filename)
+ # rename file to new filename and change to lowercase
+ os.rename(
+ os.path.join(path, filename), os.path.join(path, new_filename.lower())
+ )
+
+if __name__ == "__main__":
+
+ clean_file_names(sys.argv[1])
\ No newline at end of file
diff --git a/prepare_data/preparare_train_val_test.py b/prepare_data/preparare_train_val_test.py
new file mode 100644
index 0000000..19241f3
--- /dev/null
+++ b/prepare_data/preparare_train_val_test.py
@@ -0,0 +1,143 @@
+import os
+import shutil
+import numpy as np
+
+
+class PrepareTrainValTest:
+ def __init__(self,
+ data_in_path,
+ data_out_path,
+ train_size=0.7,
+ val_size=0.15,
+ test_size=0.15,
+ verbose=False
+ ):
+
+ self.data_in_path = data_in_path
+ self.data_out_path = data_out_path
+ self.train_size = train_size
+ self.val_size = val_size
+ self.test_size = test_size
+ self.verbose = verbose
+
+ def prepare_train_val_test(self):
+ """
+ Prepare train, validation, and test data sets from raw data. This function will
+ create a directory structure that looks like this:
+
+ data
+ ├── test
+ │ ├── class_1
+ │ ├── class_2
+ │ ├── class_3
+ │ └── class_4
+ ├── train
+ │ ├── class_1
+ │ ├── class_2
+ │ ├── class_3
+ │ └── class_4
+ └── val
+ ├── class_1
+ ├── class_2
+ ├── class_3
+ └── class_4
+
+ Parameters
+ ----------
+ None.
+
+ Returns
+ -------
+ None.
+
+ """
+
+ # get list of all classes
+ classes = os.listdir(self.data_in_path)
+
+ # create train, val, and test directories
+ for directory in ["train", "val", "test"]:
+ os.makedirs(os.path.join(self.data_out_path, directory), exist_ok=True)
+ for class_name in classes:
+ os.makedirs(
+ os.path.join(self.data_out_path, directory, class_name), exist_ok=True
+ )
+
+ # loop through classes
+ for class_name in classes:
+ # get list of all files in class directory
+ files = os.listdir(os.path.join(self.data_in_path, class_name))
+ # shuffle files
+ np.random.shuffle(files)
+ # get number of files in class directory
+ num_files = len(files)
+ # get number of files for each data set
+ num_train = int(num_files * self.train_size)
+ num_val = int(num_files * self.val_size)
+ num_test = int(num_files * self.test_size)
+ # loop through files
+ for i, file in enumerate(files):
+ # copy file to train directory
+ if i < num_train:
+ shutil.copy(
+ os.path.join(self.data_in_path, class_name, file),
+ os.path.join(self.data_out_path, "train", class_name, file),
+ )
+ # copy file to val directory
+ elif i < num_train + num_val:
+ shutil.copy(
+ os.path.join(self.data_in_path, class_name, file),
+ os.path.join(self.data_out_path, "val", class_name, file),
+ )
+ # copy file to test directory
+ else:
+ shutil.copy(
+ os.path.join(self.data_in_path, class_name, file),
+ os.path.join(self.data_out_path, "test", class_name, file),
+ )
+
+
+if __name__ == "__main__":
+ # use argparse to get command line arguments
+ import argparse
+
+ parser = argparse.ArgumentParser(
+ description="Prepare train, validation, and test data sets from raw data."
+ )
+ parser.add_argument(
+ '-i',
+ "--data_in_path",
+ type=str,
+ required=True,
+ help="Path to directory containing raw data."
+ )
+ parser.add_argument(
+ '-o',
+ "--data_out_path",
+ type=str,
+ required=True,
+ help="Path to directory to save train, validation, and test data."
+ )
+ parser.add_argument(
+ "--verbose",
+ type=bool,
+ default=False,
+ help="Enable verbose mode."
+ )
+ args = parser.parse_args()
+
+ # create instance of PrepareTrainValTest class
+
+ prepare_train_val_test = PrepareTrainValTest(
+ data_in_path=args.data_in_path,
+ data_out_path=args.data_out_path,
+ verbose=args.verbose
+ )
+
+ # prepare train, validation, and test data sets
+ prepare_train_val_test.prepare_train_val_test()
+
+
+
+
+
\ No newline at end of file
--
GitLab