From e32fbfbeb72dacf6ced66679473be098995a55d4 Mon Sep 17 00:00:00 2001 From: Guillaume Slizewicz Date: Thu, 12 May 2022 10:47:24 +0200 Subject: [PATCH] adding jupyter notebook --- commenting_code_model/random forest.ipynb | 426 ++++++++++++++++++++++ 1 file changed, 426 insertions(+) create mode 100644 commenting_code_model/random forest.ipynb diff --git a/commenting_code_model/random forest.ipynb b/commenting_code_model/random forest.ipynb new file mode 100644 index 0000000..5091257 --- /dev/null +++ b/commenting_code_model/random forest.ipynb @@ -0,0 +1,426 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Random Forest" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from random import seed\n", + "from random import randrange\n", + "from csv import reader\n", + "from math import sqrt\n", + "import json\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + " # Load a CSV file. Definition of the function to read the csv and create dataset here\n", + "def load_csv(filename):\n", + "\tdataset = list()\n", + "\twith open(filename, 'r') as file:\n", + "\t\tcsv_reader = reader(file)\n", + "\t\tfor row in csv_reader:\n", + "\t\t\tif not row:\n", + "\t\t\t\tcontinue\n", + "\t\t\tdataset.append(row)\n", + "\treturn dataset\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Convert string column to float\n", + "def str_column_to_float(dataset, column):\n", + "\tfor row in dataset:\n", + "\t\trow[column] = float(row[column].strip())\n", + " \n", + "# Convert string column to integer\n", + "def str_column_to_int(dataset, column):\n", + "\tclass_values = [row[column] for row in dataset] # extract the values of the column (here the classes of the dataset, mine and rocks)\n", + "\tunique = set(class_values) # calculate how many unique class values there are and store them into a set: a list with unique values\n", + "\tlookup = dict() #create a dictionnary\n", + "\tfor i, value in enumerate(unique): # loops through the set / enumerate gives you a tuple with an index number and a value /common way to get indexes from a list\n", + "\t\tlookup[value] = i # the key of the dictonnary is the value: mine or rock; and the value is a number: 0 or 1\n", + "\tfor row in dataset: # loops through the rows of the dataset\n", + "\t\trow[column] = lookup[row[column]] #replaces the value of the column: rock or mine, with the index value: 0 or 1;\n", + "\treturn lookup # the code returns the lookup table\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# Split a dataset into k folds\n", + "def cross_validation_split(dataset, n_folds):\n", + "\tdataset_split = list() #create a list \n", + "\tdataset_copy = list(dataset) # creates a list of the dataset. You could use the copy library\n", + "\tfold_size = int(len(dataset) / n_folds) # the size of the fold is equal to the length of the dataset divided by the amount of folds\n", + "\tprint(\"fold size:\")\n", + "\tprint(fold_size) \n", + "\tfor i in range(n_folds): #loops the amount of folds : generate another list with numbers from 0 up to n_fold\n", + "\t\tfold = list() #create a list\n", + "\t\twhile len(fold) < fold_size: # as long as the length of the list is inferior to the defined fold size\n", + "\t\t\tindex = randrange(len(dataset_copy)) # return a random integer between 0 and the total length of the dataset and store it in index\n", + "\t\t\tfold.append(dataset_copy.pop(index)) # append an observation at the index and removes it from the dataset\n", + "\t\tdataset_split.append(fold) # append the fold to the list dataset_split\n", + "\t\t#print(\"______________\")\n", + "\t\t#print(\"dataset split:\")\n", + "\t\t#print(dataset_split)\n", + "\t\t#print(\"______________\")\n", + "\treturn dataset_split #return the dataset_split, a list of folds\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# Calculate accuracy percentage\n", + "def accuracy_metric(actual, predicted):\n", + "\tcorrect = 0 #create a correct variable\n", + "\tfor i in range(len(actual)): # loops up to the length of the actual list\n", + "\t\tif actual[i] == predicted[i]: # compares the actual vs the predicted\n", + "\t\t\tcorrect += 1 #if correct add one to the correct variable. Count the number of correct guesses\n", + "\treturn correct / float(len(actual)) * 100.0 #gives a percentage by dividing the correct guesses by the length of the actual classes and multiply by a hundred\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# Evaluate an algorithm using a cross validation split\n", + "def evaluate_algorithm(dataset, algorithm, n_folds, *args):\n", + "\tfolds = cross_validation_split(dataset, n_folds) #return the list of folds\n", + "\tscores = list() #creates a list called scores\n", + "\tfor fold in folds: #loops through the folds\n", + "\t\ttrain_set = list(folds) #creates a copy of the list of folds \n", + "\t\ttrain_set.remove(fold) #remove one fold: for testing?\n", + "\t\ttrain_set = sum(train_set, []) # concatenate all folds, a list of lists, into one list. can be done with itertools.chain\n", + "\t\ttest_set = list() # create another list\n", + "\t\tfor row in fold: # iterates through fold\n", + "\t\t\trow_copy = list(row) # creates a copy of the list\n", + "\t\t\ttest_set.append(row_copy) # append the list to the test_set\n", + "\t\t\trow_copy[-1] = None # set the classification to none. Changes the last column to none.\n", + "\t\tpredicted = algorithm(train_set, test_set, *args) # what is doing the prediction takes for argument the train and test set and returns prediction for the test set\n", + "\t\tactual = [row[-1] for row in fold] # list comprehension: list of actual classes from fold.\n", + "\t\taccuracy = accuracy_metric(actual, predicted) # function that compares the actual vs the predicted to give an idea of the accuracy of the prediction\n", + "\t\tscores.append(accuracy) #append the accuracy to the list of scores\n", + "\treturn scores\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "\n", + "# Split a dataset based on an attribute/feature and an attribute/feature value\n", + "def test_split(index, value, dataset):\n", + "\tleft, right = list(), list() # create two lists for each side\n", + "\tfor row in dataset: #iterate through each row of the dataset\n", + "\t\tif row[index] < value: #if the feature value of the current row is below the feature value given\n", + "\t\t\tleft.append(row) # append it to the left list\n", + "\t\telse:\n", + "\t\t\tright.append(row) # append it to the right list\n", + "\treturn left, right # return the two lists\n", + " \n", + "# Calculate the Gini index for a split dataset\n", + "def gini_index(groups, classes):\n", + "\t# count all samples at split point\n", + "\tn_instances = float(sum([len(group) for group in groups])) #counting the total number of instances into float for divisions\n", + "\t# sum weighted Gini index for each group\n", + "\tgini = 0.0 #gini variable\n", + "\tfor group in groups: #for each of the group\n", + "\t\tsize = float(len(group)) #number of instances in each group\n", + "\t\t# avoid divide by zero\n", + "\t\tif size == 0:\n", + "\t\t\tcontinue\n", + "\t\tscore = 0.0\n", + "\t\t# score the group based on the score for each class\n", + "\t\tfor class_val in classes:\n", + "\t\t\tp = [row[-1] for row in group].count(class_val) / size #count the number of instances for the current class in the group and divide it by the total size of the group\n", + "\t\t\tscore += p * p #amplifying the difference (exponential ?)\n", + "\t\t# weight the group score by its relative size\n", + "\t\tgini += (1.0 - score) * (size / n_instances) # substract the score from 1 and multiply it by the relative size of the group compared to the dataset\n", + "\treturn gini\n", + " \n", + "# Select the best split point for a dataset\n", + "def get_split(dataset, n_features):\n", + "\tclass_values = list(set(row[-1] for row in dataset)) # creates a list of the set for the class values. Here encoded as 1 and 0 . We already did it before\n", + "\tb_index, b_value, b_score, b_groups = 999, 999, 999, None # assign values to variables\n", + "\tfeatures = list() #creates a list. \n", + "\twhile len(features) < n_features: # as long as features is smaller that the actual number of desirable features= n_features\n", + "\t\tindex = randrange(len(dataset[0])-1) # create a random number between 0 and the number of columns-1= minus the class\n", + "\t\tif index not in features: # if the column names is not already in the features\n", + "\t\t\tfeatures.append(index) # append the column name\n", + "\tfor index in features: # for each column name(=index name) in features\n", + "\t\tfor row in dataset: # for each row of the dataset\n", + "\t\t\tgroups = test_split(index, row[index], dataset) # get two lists. very computationnaly heavy. Why not do the ordering first?\n", + "\t\t\tgini = gini_index(groups, class_values) # get the gini value\n", + "\t\t\tif gini < b_score: #if the gini value is smaller than b_score (b for best?). this should always be true for the first operation. Test against the best option\n", + "\t\t\t\tb_index, b_value, b_score, b_groups = index, row[index], gini, groups # update the values for the best option\n", + "\treturn {'index':b_index, 'value':b_value, 'groups':b_groups} #return the best option in the form of a dictionnary\n", + " \n", + "# Create a terminal node value\n", + "# Returns the most popular value in the group\n", + "def to_terminal(group):\n", + "\toutcomes = [row[-1] for row in group] #takes the class of the group elements and put it in a list\n", + "\treturn max(set(outcomes), key=outcomes.count) #return the class that appears the most. set is counting the different class and the key is counting the number of occurences of this class. SPOOKY AND DENSE\n", + " \n", + "# Create child splits for a node or make a terminal (node)\n", + "def split(node, max_depth, min_size, n_features, depth):\n", + "\tleft, right = node['groups'] #get the groups using the dictionnary node with the key groups\n", + "\tdel(node['groups']) #delete dictionnary element with key groups\n", + "\t# check for a no split\n", + "\tif not left or not right: #if one of the group is empty\n", + "\t\tnode['left'] = node['right'] = to_terminal(left + right) #call this function\n", + "\t\treturn \n", + "\t# check for max depth\n", + "\tif depth >= max_depth: # checking if level is bigger or equal than maximum level of nodes\n", + "\t\tnode['left'], node['right'] = to_terminal(left), to_terminal(right) #if yes, so both nodes are terminal nodes\n", + "\t\treturn\n", + "\t# process left child\n", + "\tif len(left) <= min_size: # if the group is smaller or equal than the minimum size for a group\n", + "\t\tnode['left'] = to_terminal(left) #left node is a terminal node\n", + "\telse:\n", + "\t\tnode['left'] = get_split(left, n_features) #create another split, another node from which to separate in two groups with a subset of a dataset\n", + "\t\tsplit(node['left'], max_depth, min_size, n_features, depth+1) #recursion -> calls itself\n", + "\t# process right child\n", + "\tif len(right) <= min_size:\n", + "\t\tnode['right'] = to_terminal(right)\n", + "\telse:\n", + "\t\tnode['right'] = get_split(right, n_features)\n", + "\t\tsplit(node['right'], max_depth, min_size, n_features, depth+1)\n", + " \n", + "# Build a decision tree\n", + "def build_tree(train, max_depth, min_size, n_features):\n", + "\troot = get_split(train, n_features) #get the index, value and group (?) for the split. It's a dictionnary\n", + "\tsplit(root, max_depth, min_size, n_features, 1) #depth is one\n", + " # root is Node\n", + " # Node: { index: int, value: float, left: Node|TerminalNode, right: Node|TerminalNode }\n", + " # TerminalNode: { index: int, value: float, left: int(class), right: int(class) }\n", + "\treturn root #dictionnary with index, value, left, right\n", + " \n", + "# Make a prediction with a decision tree\n", + "def predict(node, row):\n", + "\tif row[node['index']] < node['value']: #if the feature value of the row is smaller of the feature value of the node\n", + "\t\tif isinstance(node['left'], dict): # is it a node or a terminal node(children are not node)\n", + "\t\t\treturn predict(node['left'], row) #recursion if a node. the function calls itselft on the following left node\n", + "\t\telse:\n", + "\t\t\treturn node['left'] # result if a terminal node\n", + "\telse:\n", + "\t\tif isinstance(node['right'], dict): # is it a node or a terminal node(children are not node)\n", + "\t\t\treturn predict(node['right'], row)#recursion if a node. the function calls itselft on the following right node\n", + "\t\telse:\n", + "\t\t\treturn node['right']# result if a terminal node\n", + " \n", + "# Create a random subsample from the dataset with replacement\n", + "def subsample(dataset, ratio):\n", + "\tsample = list() #creates a list\n", + "\tn_sample = round(len(dataset) * ratio) # rounds the multiplication of the length of the dataset with the sample.size: here 1. \n", + "\twhile len(sample) < n_sample: # loop up until the length of the sample is the length of n_sample\n", + "\t\tindex = randrange(len(dataset)) #take a random number from 0 up to length of the dataset\n", + "\t\tsample.append(dataset[index]) # append a sample whith this index\n", + "\treturn sample # return the list of sub-samples\n", + " \n", + "# Make a prediction with a list of bagged trees\n", + "def bagging_predict(trees, row): \n", + "\tpredictions = [predict(tree, row) for tree in trees] # we run the prediction in each tree, this gives a list of predictions of classes/votes. THE TREES ARE VOTING.\n", + "\treturn max(set(predictions), key=predictions.count) # we count the class with the maximum votes and return it as prediction\n", + " \n", + "# Random Forest Algorithm\n", + "def random_forest(train, test, max_depth, min_size, sample_size, n_trees, n_features):\n", + "\ttrees = list() #create a list of trees\n", + "\tfor i in range(n_trees): \n", + "\t\tsample = subsample(train, sample_size) # create subsamples \n", + "\t\ttree = build_tree(sample, max_depth, min_size, n_features) #build the tree\n", + "\t\ttrees.append(tree)#append the list \n", + "\twith open(\"model.json\", \"w\") as out_file: \n", + "\t\tjson.dump(trees, out_file, indent = 6)\n", + "\tpredictions = [bagging_predict(trees, row) for row in test] # testing with test data, running the prediction on every row. THE FOREST VOTES ON EVERY ROW\n", + "\treturn(predictions) # we return the predicted class of each of the rows as a list. THE PREDICTIONS OF THE FOREST\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "from random import random" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# Test the random forest algorithm\n", + "seed(2) #put the random generator in a certain state -> makes it deterministic otherwise python takes the time of the computer\n", + "#print(random())\n", + "#print(random())\n", + "#seed(2)\n", + "#print(random())\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "# load and prepare data\n", + "filename = 'iris.csv'\n", + "#filename = 'iris.csv'\n", + "\n", + "dataset = load_csv(filename)\n", + "#print(dataset)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "# convert string attributes to integers\n", + "for i in range(0, len(dataset[0])-1):\n", + "\tstr_column_to_float(dataset, i)\n", + "# convert class column to integers\n", + "#str_column_to_int(dataset, len(dataset[0])-1) #this function loops through the rows and transforms the words mine and roch into 1 and 0\n", + "#print(dataset)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "# evaluate algorithm\n", + "n_folds = 5 #the data is randomly sampled into 5 subsamples, one is kept for testing the 4 else are used for training.\n", + "max_depth = 10 #\n", + "min_size = 1\n", + "sample_size = 1.0 #fixed, can be changed to reduces the amount of subsampling, if it is smaller than one.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fold size:\n", + "30\n", + "Trees: 1\n", + "Scores: [100.0, 93.33333333333333, 90.0, 100.0, 93.33333333333333]\n", + "Mean Accuracy: 95.333%\n", + "fold size:\n", + "30\n", + "Trees: 5\n", + "Scores: [96.66666666666667, 96.66666666666667, 100.0, 93.33333333333333, 93.33333333333333]\n", + "Mean Accuracy: 96.000%\n", + "fold size:\n", + "30\n", + "Trees: 10\n", + "Scores: [90.0, 96.66666666666667, 96.66666666666667, 86.66666666666667, 100.0]\n", + "Mean Accuracy: 94.000%\n" + ] + } + ], + "source": [ + "n_features = int(sqrt(len(dataset[0])-1)) #it specifies the size of the feature subset for the folds, where the size is close to the square root of the total number of features\n", + "for n_trees in [1, 5, 10]: # will loop three times by taking n_trees equals one, n_trees equals five and n_trees equals ten successively.\n", + "\tscores = evaluate_algorithm(dataset, random_forest, n_folds, max_depth, min_size, sample_size, n_trees, n_features)\n", + "\tprint('Trees: %d' % n_trees)\n", + "\tprint('Scores: %s' % scores)\n", + "\tprint('Mean Accuracy: %.3f%%' % (sum(scores)/float(len(scores))))" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Iris Versicolor\n" + ] + } + ], + "source": [ + "with open (\"model.json\", \"r\") as in_file:\n", + "\ttrees=json.load(in_file)\n", + "\tprediction=bagging_predict(trees,dataset[55])\n", + "\tprint(prediction)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}