Anul 3 Semestrul 1

Anul 3/Semestrul 1/Proiect Colectiv/StoryPointEstimator/.gitignore

@@ -0,0 +1,3 @@
+__pycache__/
+.env
+experiments/

Anul 3/Semestrul 1/Proiect Colectiv/StoryPointEstimator/README.md

@@ -0,0 +1,88 @@
+# Story Point Estimator
+
+This repository contains code for training and evaluating and exporting models for story point estimation.
+
+The dataset used for training is the [IEEE TSE2018 dataset](https://github.com/jai2shukla/JIRA-Estimation-Prediction/tree/master/storypoint/IEEE%20TSE2018/dataset) which contains user stories from 16 open-source projects from 9 different organizations.
+
+
+Currently, the following models are supported:
+- [x] [DistilBERT](https://arxiv.org/abs/1910.01108)
+
+
+## Setup
+
+#### Create and activate conda environment
+
+```bash
+conda create --name spestimator python=3.8
+conda activate spestimator
+```
+
+#### Install dependencies
+
+```bash
+pip install torch==2.4.1 --index-url https://download.pytorch.org/whl/cu124
+pip install -r requirements.txt
+```
+
+#### Setup dotenv
+
+Create a `.env` file in the root directory and add the following environment variables:
+
+```bash
+DATA_PATH=<path to data directory>
+```
+
+
+## Usage
+
+#### Train models for story point estimation
+
+See training options
+
+```bash
+python train.py --help
+```
+
+Train a model with your desired options while also seing validation results
+
+```bash
+python train.py <options>
+```
+
+Users can see in the experiments folder the results inside a subfolder with their experiment's name. The folder contains: <br>
+        - a checkpoints folder where the results of each epoch and their according performance metrics are stored <br>
+        - a params.json file with the experiment's parameters <br>
+        - a log file with the experiment's logs regarding the training process <br>
+
+
+#### Evaluate existing models
+
+See evaluation options
+
+```bash
+python eval.py --help
+```
+
+Evaluate a model with your desired options
+
+```bash
+python eval.py <options>
+```
+
+
+#### Export models
+
+See export options
+
+```bash
+python export.py --help
+```
+
+Export a model with your desired options
+
+```bash
+python export.py <options>
+```
+
+Exports the model and tokenizer to the specified directory. The <b>model</b> is stored in a <b>.safetensors</b> file (for python users) and an <b>.onnx</b> file for other languages, and the <b>tokenizer</b> is stored in a <b>vocab.txt</b> file and 4 json files: <b>config.json, special_tokens_map.json, tokenizer_config.json, and tokenizer.json</b>.

Anul 3/Semestrul 1/Proiect Colectiv/StoryPointEstimator/requirements.txt

@@ -0,0 +1,6 @@
+transformers==4.46.2
+python-dotenv==1.0.1
+pandas==2.0.3
+onnx==1.17.0
+onnxruntime==1.19.2
+scikit-learn==1.3.2

Anul 3/Semestrul 1/Proiect Colectiv/StoryPointEstimator/src/data_loader/__init__.py

@@ -0,0 +1,7 @@
+from .dataframe_generator import DFGenerator
+from .storypoint_dataset import StoryPointDataset
+
+__all__ = [
+    'DFGenerator',
+    'StoryPointDataset'
+]

Anul 3/Semestrul 1/Proiect Colectiv/StoryPointEstimator/src/data_loader/dataframe_generator.py

@@ -0,0 +1,81 @@
+import os
+import glob
+from dotenv import load_dotenv
+
+import pandas as pd
+
+
+class DFGenerator:
+
+    '''
+    Class to generate dataframes from csv files in the data_path directory.
+    '''
+
+    def __init__(self, eval_split = 0.1, random_seed = 42):
+
+        load_dotenv()
+        self.data_path = os.getenv('DATA_PATH')
+        self.eval_split = eval_split
+        self.random_seed = random_seed
+
+    
+    def create_dataframes(self):
+
+        '''
+        Reads all csv files in the data_path directory and creates a dataframe out of them.
+        The dataframe is then filtered to only include storypoints that are fibonacci numbers <= 13.
+
+        Dataframe structure: (text: str, label: int)
+            - text: made by concatenating the title and description columns
+            - label: position in fibonacci sequence of the storypoint value
+
+        Returns:
+            train_df, eval_df: dataframes containing the training and evaluation data
+            if eval_split is 0, eval_df will be None
+        '''
+
+        
+        data_files = glob.glob(self.data_path + '/*.csv')
+        data = []
+
+        for filename in data_files:
+            df = pd.read_csv(filename, index_col=None, header=0)
+            data.append(df)
+            storypoint_df = pd.concat(data, axis=0, ignore_index=True)
+
+        storypoint_df['storypoint'] = storypoint_df['storypoint'].apply(lambda x: 14 if x > 13 else x)
+        
+        labels = [1, 2, 3, 5, 8, 13, 14]  # 14 means value too high, ticket needs to be split
+        storypoint_df = storypoint_df[storypoint_df['storypoint'].isin(labels)]
+
+        storypoint_df['text'] = storypoint_df['title'] + ' ' + storypoint_df['description'].fillna('')
+        storypoint_df = storypoint_df[['text', 'storypoint']]
+
+        label_mapping = {1: 0, 2: 1, 3: 2, 5: 3, 8: 4, 13: 5, 14: 6}
+        storypoint_df['label'] = storypoint_df['storypoint'].map(label_mapping)
+        storypoint_df = storypoint_df[['text', 'label']]
+
+        if self.eval_split == 0:
+            return storypoint_df, None
+        
+        elif self.eval_split >=1:
+            return None, storypoint_df
+        
+        validation_df = pd.DataFrame()
+        label_counts_total = self._label_counts(storypoint_df, 'label')
+
+        for label in label_counts_total.index:
+            label_data = storypoint_df[storypoint_df['label'] == label]
+            validation_size = int(len(label_data) * self.eval_split)
+            validation_sample = label_data.sample(n=validation_size, random_state=self.random_seed)
+            validation_df = pd.concat([validation_df, validation_sample], ignore_index=True)
+
+        train_df = storypoint_df[~storypoint_df.index.isin(validation_df.index)].copy()
+
+        return train_df, validation_df
+
+    
+    def _label_counts(self, df, column_name):
+        
+        label_counts = df[column_name].value_counts()
+        return label_counts

Anul 3/Semestrul 1/Proiect Colectiv/StoryPointEstimator/src/data_loader/storypoint_dataset.py

@@ -0,0 +1,36 @@
+import torch
+from torch.utils.data import Dataset
+
+
+class StoryPointDataset(Dataset):
+    
+    '''
+    Dataset class for the storypoint data.
+    Requires a dataframe with columns 'text' and 'label' and a tokenizer.
+    '''
+
+    def __init__(self, dataframe, tokenizer, max_length=128):
+        self.dataframe = dataframe
+        self.tokenizer = tokenizer
+        self.max_length = max_length
+
+
+    def __len__(self):
+
+        return len(self.dataframe)
+    
+
+    def __getitem__(self, idx):
+        
+        text = self.dataframe.iloc[idx]['text']
+        label = self.dataframe.iloc[idx]['label']
+        inputs = self.tokenizer(
+            text,
+            max_length=self.max_length,
+            padding='max_length',
+            truncation=True,
+            return_tensors='pt'
+        )
+        item = {key: val.squeeze(0) for key, val in inputs.items()}
+        item['labels'] = torch.tensor(label)
+        return item

Anul 3/Semestrul 1/Proiect Colectiv/StoryPointEstimator/src/eval.py

@@ -0,0 +1,71 @@
+from dotenv import load_dotenv
+import argparse
+from tqdm import tqdm
+
+import torch
+from torch.utils import data
+from data_loader import DFGenerator, StoryPointDataset
+from models import get_model_and_tokenizer
+
+
+def parse_args() -> argparse.Namespace:
+
+    parser = argparse.ArgumentParser(description='Model evaluation details')
+
+    # Dataset details
+    parser.add_argument('--eval_split', type=float, default=0.2, help='Evaluation split')
+    parser.add_argument('--random_seed', type=int, default=42, help='Random seed')
+    parser.add_argument('--max_length', type=int, default=128, help='Maximum number of tokens in input sequence')
+
+    # Model details
+    parser.add_argument('--checkpoint', type=str, default=None, help='Checkpoint to evaluate')
+    return parser.parse_args()
+
+
+class Evaluator():
+
+    def __init__(self, args):
+
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+        self.model, self.tokenizer = get_model_and_tokenizer(args.checkpoint)
+        self.model.to(self.device)
+
+        df_generator = DFGenerator(eval_split=args.eval_split, random_seed=args.random_seed)
+
+        _, val_df = df_generator.create_dataframes()
+        if val_df is None:
+            raise ValueError('No validation data available')
+
+        val_dataset = StoryPointDataset(val_df, self.tokenizer, max_length=args.max_length)
+        self.val_loader = data.DataLoader(val_dataset, batch_size=1)
+
+
+    def evaluate(self):
+
+        self.model.eval()
+        correct_predictions = 0
+        total_predictions = 0
+
+        for batch in tqdm(self.val_loader, desc=f'Validation'):
+            inputs = {k: v.to(self.model.device) for k, v in batch.items()}
+            with torch.no_grad():
+                outputs = self.model(**inputs)
+
+            _, predicted = torch.max(outputs.logits, 1)
+            correct_predictions += (predicted == batch['labels'].to(self.model.device)).sum().item()
+            total_predictions += len(batch['labels'])
+
+        accuracy = correct_predictions / total_predictions
+        validation_message = f'Validation Accuracy: {accuracy:.4f}'
+
+        print(validation_message)
+
+
+if __name__ == '__main__':
+    
+    load_dotenv()
+    args = parse_args()
+
+    evaluator = Evaluator(args)
+    evaluator.evaluate()

Anul 3/Semestrul 1/Proiect Colectiv/StoryPointEstimator/src/export.py

@@ -0,0 +1,77 @@
+import argparse
+
+from models import get_model_and_tokenizer
+from data_loader import DFGenerator, StoryPointDataset
+
+import numpy as np
+import torch
+import onnx
+import onnxruntime
+
+
+def parse_args() -> argparse.Namespace:
+
+    parser = argparse.ArgumentParser(description='Model export details')
+    parser.add_argument('--checkpoint', type=str, default=None, help='Checkpoint to export')
+    parser.add_argument('--output_dir', type=str, default='.', help='Output directory for exported model')
+    parser.add_argument('--model_name', type=str, default='storypoint_estimator', help='Model name')
+    return parser.parse_args()
+
+
+def export_model(args):
+
+    model, tokenizer = get_model_and_tokenizer(args.checkpoint)
+    model.save_pretrained(args.output_dir)
+    tokenizer.save_pretrained(args.output_dir)
+
+    x, _ = DFGenerator().create_dataframes()
+    x = x.sample(1)
+    dataset = StoryPointDataset(x, tokenizer)
+    sample_data = dataset[0]
+
+    dummy_input = dataset[0]['input_ids'].unsqueeze(0)
+    attention_mask = dataset[0]['attention_mask'].unsqueeze(0)
+
+
+    model.eval()
+    with torch.no_grad():
+        output = model(dummy_input, attention_mask=attention_mask)
+
+    torch.onnx.export(model,
+                      (dummy_input, attention_mask),
+                      f'{args.output_dir}/{args.model_name}.onnx',
+                      opset_version=14,
+                      do_constant_folding=True,
+                      input_names=['input_ids', 'attention_mask'],
+                      output_names=['output'],
+                      dynamic_axes={
+                            'input_ids': {0: 'batch_size'},
+                            'attention_mask': {0: 'batch_size'},
+                            'output': {0: 'batch_size'}
+                        })
+    
+
+    onnx_model = onnx.load(f'{args.output_dir}/{args.model_name}.onnx')
+    onnx.checker.check_model(onnx_model)
+
+    ort_session = onnxruntime.InferenceSession(f'{args.output_dir}/{args.model_name}.onnx', providers=['CPUExecutionProvider'])
+
+    def to_numpy(tensor):
+        return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()
+
+    output_tensor = output.logits if hasattr(output, 'logits') else output.last_hidden_state
+
+    ort_inputs = {
+        ort_session.get_inputs()[0].name: to_numpy(dummy_input),
+        ort_session.get_inputs()[1].name: to_numpy(attention_mask)
+    }
+    ort_outs = ort_session.run(None, ort_inputs)
+
+    np.testing.assert_allclose(to_numpy(output_tensor), ort_outs[0], rtol=1e-03, atol=1e-05)
+    print("Exported model has been validated successfully!")
+
+
+if __name__ == '__main__':
+    
+    args = parse_args()
+    export_model(args)

Anul 3/Semestrul 1/Proiect Colectiv/StoryPointEstimator/src/models/__init__.py

@@ -0,0 +1,3 @@
+from .distilbert_classifier import get_model_and_tokenizer
+
+__all__ = ['get_model_and_tokenizer']

Anul 3/Semestrul 1/Proiect Colectiv/StoryPointEstimator/src/models/distilbert_classifier.py

@@ -0,0 +1,17 @@
+import torch
+from transformers import AutoTokenizer, DistilBertForSequenceClassification
+
+
+def get_model_and_tokenizer(checkpoint=None):
+    
+    model = DistilBertForSequenceClassification.from_pretrained(
+                'distilbert-base-uncased',
+                num_labels=7
+            )
+    
+    if checkpoint:
+        print(f"Loading checkpoint: {checkpoint}")
+        model.load_state_dict(torch.load(checkpoint, weights_only=True))
+
+    tokenizer = AutoTokenizer.from_pretrained('distilbert-base-uncased')
+    return model, tokenizer

Anul 3/Semestrul 1/Proiect Colectiv/StoryPointEstimator/src/train.py

@@ -0,0 +1,220 @@
+import os
+from dotenv import load_dotenv
+import argparse
+import logging
+import json
+from tqdm import tqdm
+
+import torch
+from torch.utils import data
+from sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix
+import numpy as np
+
+from data_loader import DFGenerator, StoryPointDataset
+from models import get_model_and_tokenizer
+from utils import serialize_metrics
+
+
+def parse_args() -> argparse.Namespace:
+
+    parser = argparse.ArgumentParser(description='Model training details')
+
+    # Dataset details
+    parser.add_argument('--eval_split', type=float, default=0.1, help='Evaluation split')
+    parser.add_argument('--random_seed', type=int, default=42, help='Random seed')
+    parser.add_argument('--max_length', type=int, default=128, help='Maximum number of tokens in input sequence')
+
+    # Model training details
+    parser.add_argument('--checkpoint', type=str, default=None, help='Checkpoint to resume training')
+    parser.add_argument('--epochs', type=int, default=20, help='Number of epochs')
+    parser.add_argument('--batch_size', type=int, default=16, help='Batch size')
+    parser.add_argument('--learning_rate', type=float, default=1e-5, help='Learning rate')
+    parser.add_argument('--weight_decay', type=float, default=1e-5, help='Weight decay')
+    parser.add_argument('--eval', type=bool, default=True, help='Evaluate model after each training epoch')
+
+    # Model saving details
+    parser.add_argument('--experiment_name', type=str, default='storypoint_estimator', help='Experiment name (for checkpoint naming)')
+    return parser.parse_args()
+
+
+class Trainer():
+
+    def __init__(self, args):
+
+        self.experiment_dir = f'experiments/{args.experiment_name}'
+        self.checkpoints_dir = f'{self.experiment_dir}/checkpoints'
+
+        os.makedirs(self.experiment_dir, exist_ok=True)
+        os.makedirs(self.checkpoints_dir, exist_ok=True)
+
+        log_path = f'{self.experiment_dir}/{args.experiment_name}.log'
+
+        self.logger = logging.getLogger('training_pipeline')
+        self.logger.setLevel(logging.INFO)
+
+        file_handler = logging.FileHandler(log_path, mode='a')
+        file_handler.setLevel(logging.INFO)
+
+        self.logger.addHandler(file_handler)
+
+        params_path = f'{self.experiment_dir}/params.json'
+
+        with open(params_path, 'w') as f:
+            json.dump(vars(args), f, indent=4)
+
+
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+        self.model, self.tokenizer = get_model_and_tokenizer(args.checkpoint)
+        self.model.to(self.device)
+
+        df_generator = DFGenerator(eval_split=args.eval_split, random_seed=args.random_seed)
+
+        train_df, val_df = df_generator.create_dataframes()
+        if train_df is None:
+            raise ValueError('No training data available')
+
+        train_dataset = StoryPointDataset(train_df, self.tokenizer, max_length=args.max_length)
+        val_dataset = StoryPointDataset(val_df, self.tokenizer, max_length=args.max_length) if val_df is not None else None
+        
+        self.train_loader = data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True)
+        self.val_loader = data.DataLoader(val_dataset, batch_size=1) if val_dataset is not None else None
+
+
+        self.epochs = args.epochs
+        self.optimizer = torch.optim.AdamW(self.model.parameters(), lr=args.learning_rate, weight_decay=args.weight_decay)
+        self.lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(self.optimizer, T_max=self.epochs, eta_min=1e-7)
+
+
+        self.eval = args.eval
+        self.experiment_name = args.experiment_name
+
+
+    def train(self):
+
+        for epoch in range(self.epochs):
+            self.model.train()
+            correct_predictions = 0
+            total_predictions = 0
+            total_loss = 0
+
+            all_labels = []
+            all_predictions = []
+
+            for batch in tqdm(self.train_loader, desc=f'Epoch {epoch+1}/{self.epochs}'):
+                inputs = {k: v.to(self.model.device) for k, v in batch.items()}
+                outputs = self.model(**inputs)
+
+                loss = outputs.loss
+                loss.backward()
+
+                self.optimizer.step()
+                self.optimizer.zero_grad()
+
+                total_loss += loss.item()
+                _, predicted = torch.max(outputs.logits, 1)
+
+                correct_predictions += (predicted == batch['labels'].to(self.model.device)).sum().item()
+                total_predictions += len(batch['labels'])
+
+                all_labels.extend(batch['labels'].cpu().numpy())
+                all_predictions.extend(predicted.cpu().numpy())
+
+            all_labels = np.array(all_labels)
+            all_predictions = np.array(all_predictions)
+
+            epoch_lr = self.optimizer.param_groups[0]['lr']
+            self.lr_scheduler.step()
+
+            average_loss = total_loss / len(self.train_loader)
+            accuracy = correct_predictions / total_predictions
+            macro_f1 = f1_score(all_labels, all_predictions, average='macro')
+            weighted_f1 = f1_score(all_labels, all_predictions, average='weighted')
+            precision = precision_score(all_labels, all_predictions, average='macro')
+            recall = recall_score(all_labels, all_predictions, average='macro')
+            conf_matrix = confusion_matrix(all_labels, all_predictions)
+
+            metrics = (accuracy, macro_f1, weighted_f1, precision, recall, conf_matrix)
+
+
+            epoch_message = f"Training - Epoch {epoch+1}/{self.epochs}: Loss - {average_loss:.4f}\t\tLR - {epoch_lr:.8f}\n"
+
+            print(epoch_message)
+            self.logger.info(epoch_message)
+
+
+            if self.eval and self.val_loader:
+                self.evaluate(epoch, metrics)
+            
+            else:
+                self.save_checkpoint('train', metrics, epoch)
+
+
+
+    def evaluate(self, epoch, train_metrics = None):
+
+        if self.val_loader is None:
+            return
+        
+        self.model.eval()
+        correct_predictions = 0
+        total_predictions = 0
+
+        all_labels = []
+        all_predictions = []
+
+        for batch in tqdm(self.val_loader, desc=f'Validation - Epoch {epoch+1}/{self.epochs}'):
+            inputs = {k: v.to(self.model.device) for k, v in batch.items()}
+            with torch.no_grad():
+                outputs = self.model(**inputs)
+
+            _, predicted = torch.max(outputs.logits, 1)
+            correct_predictions += (predicted == batch['labels'].to(self.model.device)).sum().item()
+            total_predictions += len(batch['labels'])
+
+            all_labels.extend(batch['labels'].cpu().numpy())
+            all_predictions.extend(predicted.cpu().numpy())
+
+        all_labels = np.array(all_labels)
+        all_predictions = np.array(all_predictions)
+
+        accuracy = correct_predictions / total_predictions
+        macro_f1 = f1_score(all_labels, all_predictions, average='macro')
+        weighted_f1 = f1_score(all_labels, all_predictions, average='weighted')
+        precision = precision_score(all_labels, all_predictions, average='macro')
+        recall = recall_score(all_labels, all_predictions, average='macro')
+        conf_matrix = confusion_matrix(all_labels, all_predictions)
+
+        metrics = (accuracy, macro_f1, weighted_f1, precision, recall, conf_matrix)
+
+        if train_metrics is not None:
+            all_metrics = list(train_metrics)
+            all_metrics.extend(metrics)
+            metrics = tuple(all_metrics)
+
+            self.save_checkpoint('both', metrics, epoch)
+
+        else:
+            self.save_checkpoint('val', metrics, epoch)
+
+ 
+    def save_checkpoint(self, mode, metrics, epoch=0):
+
+        json_data = serialize_metrics(mode, metrics)
+
+        checkpoint_dir = f'{self.checkpoints_dir}/epoch_{epoch+1}'
+        os.makedirs(checkpoint_dir, exist_ok=True)
+
+        torch.save(self.model.state_dict(), f'{checkpoint_dir}/{self.experiment_name}_{epoch+1}.pth')
+        with open(f'{checkpoint_dir}/metrics.json', 'w') as f:
+            f.write(json_data)
+
+
+
+if __name__ == '__main__':
+    
+    load_dotenv()
+    args = parse_args()
+
+    trainer = Trainer(args)
+    trainer.train()

Anul 3/Semestrul 1/Proiect Colectiv/StoryPointEstimator/src/utils/__init__.py

@@ -0,0 +1,4 @@
+from .utils import serialize_metrics
+
+
+__all__ = ['serialize_metrics']

Anul 3/Semestrul 1/Proiect Colectiv/StoryPointEstimator/src/utils/utils.py

@@ -0,0 +1,47 @@
+import json
+import numpy as np
+
+def serialize_metrics(mode, metrics):
+    if mode == 'train':
+        json_str = '{\n'
+        json_str += '\t"training_metrics": {\n'
+        json_str += f'\t\t"accuracy": {metrics[0]},\n'
+        json_str += f'\t\t"macro_f1": {metrics[1]},\n'
+        json_str += f'\t\t"weighted_f1": {metrics[2]},\n'
+        json_str += f'\t\t"precision": {metrics[3]},\n'
+        json_str += f'\t\t"recall": {metrics[4]},\n'
+        json_str += '\t\t"confusion_matrix": ' + json.dumps(metrics[5].tolist()) + '\n'
+        json_str += '\t}\n'
+        json_str += '}'
+    elif mode == 'val':
+        json_str = '{\n'
+        json_str += '\t"validation_metrics": {\n'
+        json_str += f'\t\t"accuracy": {metrics[0]},\n'
+        json_str += f'\t\t"macro_f1": {metrics[1]},\n'
+        json_str += f'\t\t"weighted_f1": {metrics[2]},\n'
+        json_str += f'\t\t"precision": {metrics[3]},\n'
+        json_str += f'\t\t"recall": {metrics[4]},\n'
+        json_str += '\t\t"confusion_matrix": ' + json.dumps(metrics[5].tolist()) + '\n'
+        json_str += '\t}\n'
+        json_str += '}'
+    elif mode == 'both':
+        json_str = '{\n'
+        json_str += '\t"training_metrics": {\n'
+        json_str += f'\t\t"accuracy": {metrics[0]},\n'
+        json_str += f'\t\t"macro_f1": {metrics[1]},\n'
+        json_str += f'\t\t"weighted_f1": {metrics[2]},\n'
+        json_str += f'\t\t"precision": {metrics[3]},\n'
+        json_str += f'\t\t"recall": {metrics[4]},\n'
+        json_str += '\t\t"confusion_matrix": ' + json.dumps(metrics[5].tolist()) + '\n'
+        json_str += '\t},\n'
+        json_str += '\t"validation_metrics": {\n'
+        json_str += f'\t\t"accuracy": {metrics[6]},\n'
+        json_str += f'\t\t"macro_f1": {metrics[7]},\n'
+        json_str += f'\t\t"weighted_f1": {metrics[8]},\n'
+        json_str += f'\t\t"precision": {metrics[9]},\n'
+        json_str += f'\t\t"recall": {metrics[10]},\n'
+        json_str += '\t\t"confusion_matrix": ' + json.dumps(metrics[11].tolist()) + '\n'
+        json_str += '\t}\n'
+        json_str += '}'
+    
+    return json_str