code_compile.rmd

---
title: "Final Project"
author: "Soobin Choi"
date: "2022-10-01"
output: 
  github_document: 
    toc: TRUE
---

# Developmental process of L2 - English and Korean

```{r setup, message=FALSE}
knitr::opts_chunk$set(echo=TRUE, include=TRUE, comment="")
library(tidyverse)
library(tidytext)
library(repurrrsive)
library(quanteda)
library(magrittr)
library(ggplot2)
```


## Korean Learners' Corpus (KLC)

### Data Import


```{r}
KLC <- read_tsv(file = "https://github.com/jungyeul/korean-learner-corpus/raw/main/data/kyunghee_v2.tsv", locale(encoding = "UTF-8")) 

KLC <- KLC %>% 
  rename(ID = X1,
         Nationality = X2,
         Gender = X3,
         Topic = X4,
         Text = X5,
         Morphemes = X6,
         Level = X7,
         Score = X8)

KLC_clean <- KLC %>% 
  filter(Level %in% c("A1", "A2", "B1", "B2", "C1", "C2"))

KLC_eng <- KLC_clean %>% 
  filter(Nationality %in% c("미국", "영국", "호주", "필리핀", "싱가포르", "인도", "르완다"))

KLC_clean <- KLC_eng %>%
  select(ID, Morphemes) %>% 
  map(~ str_split(., " (\\+)?")) %>% 
  as_data_frame() %>% 
  rename(num_token = Morphemes) %>% 
  unnest(ID) %>% 
  left_join(KLC_eng, .,  by="ID") %>% 
  relocate(num_token, .after = ID) %>% 
  select(-c(Gender, Topic))
```


#### 1. Lexical diversity

```{r}
KLC_numtok <- KLC_clean %>% 
  unnest(num_token) %>% 
  mutate_all(funs(str_replace(., "A1", "2"))) %>% 
  mutate_all(funs(str_replace(., "A2", "2"))) %>% 
  mutate_all(funs(str_replace(., "B1", "3"))) %>% 
  mutate_all(funs(str_replace(., "B2", "4"))) %>% 
  mutate_all(funs(str_replace(., "C1", "5"))) %>% 
  mutate_all(funs(str_replace(., "C2", "5"))) %>% 
  group_by(ID) %>% 
  count(Level, name = "num_token")

KLC_numdist <- KLC_clean %>% 
  unnest(num_token) %>% 
  mutate_all(funs(str_replace(., "A1", "2"))) %>% 
  mutate_all(funs(str_replace(., "A2", "2"))) %>% 
  mutate_all(funs(str_replace(., "B1", "3"))) %>% 
  mutate_all(funs(str_replace(., "B2", "4"))) %>% 
  mutate_all(funs(str_replace(., "C1", "5"))) %>% 
  mutate_all(funs(str_replace(., "C2", "5"))) %>% 
  group_by(ID) %>% 
  summarize(num_dist = n_distinct(num_token))


KLC_TTR <- left_join(KLC_numdist, KLC_numtok, by = "ID") %>% 
   mutate(TTR = round((num_dist / num_token),4)) %>% 
   rename(mean_text_len = num_token) %>% 
   select(-ID) %>% 
   group_by(Level) %>% 
   summarize(mean_text_len = mean(mean_text_len),
             TTR = round(mean(TTR), 4))

```



#### 2. Syntactic Complexity


```{r, error=TRUE}

KLC_sent <- KLC_clean %>% 
  unnest_tokens(Sentence, Text, "sentences") 

KLC_morph <- KLC_sent %>% 
  unnest(num_token) %>% 
  mutate(sent_num = lag(num_token) %>% 
           str_detect("/EF$") %>% 
           replace_na(FALSE) %>% 
           cumsum() %>% 
           add(1),
         .after = num_token)

KLC_syncom <- KLC_morph %>% 
  select(-Nationality, -Morphemes, -Score, -Sentence) %>% 
  group_by(sent_num) %>% 
  mutate(sent_len = length(ID)) %>% 
  mutate_all(funs(str_replace(., "A1", "A2"))) %>% 
  mutate_all(funs(str_replace(., "C2", "C1"))) %>% 
  mutate(sent_len = as.numeric(sent_len)) %>% 
  group_by(Level) %>% 
  summarize(mean_sent_len = mean(sent_len))
  


```


### KLC final

```{r}
KLC_TTR
KLC_syncom
```



## PELIC


### Data Import


```{r}
PELIC_ans <- read_csv("https://github.com/ELI-Data-Mining-Group/PELIC-dataset/raw/master/corpus_files/answer.csv")

PELIC_crs <- read_csv("https://github.com/ELI-Data-Mining-Group/PELIC-dataset/raw/master/corpus_files/course.csv")

PELIC_id <- read_csv("https://github.com/ELI-Data-Mining-Group/PELIC-dataset/raw/master/corpus_files/student_information.csv")

PELIC_scr <- read_csv("https://github.com/ELI-Data-Mining-Group/PELIC-dataset/raw/master/corpus_files/test_scores.csv")

PELIC_ques <- read_csv("https://github.com/ELI-Data-Mining-Group/PELIC-dataset/raw/master/corpus_files/question.csv")
```

### Data manipulation

** sorting out columns needed
```{r}

PELIC_scr1 <- PELIC_scr %>% 
  select(anon_id, MTELP_Conv_Score, Writing_Sample)

PELIC_crs1 <- PELIC_crs %>% 
  select(course_id, level_id)

PELIC_ans1 <- PELIC_ans %>%
  select(anon_id, course_id, question_id, text_len, text, tokens, tok_lem_POS)
  
PELIC_id1 <- PELIC_id %>% 
  select(anon_id, native_language)

PELIC_ques1 <- PELIC_ques %>% 
  select(question_id, question_type_id)
```



** Joining all the columns in one dataframe

```{r}
PELIC1 <- left_join(PELIC_ans1, PELIC_crs1, by = "course_id") %>%  
  relocate(c(level_id), .after = course_id)
PELIC2 <- left_join(PELIC1, PELIC_id1,by = "anon_id")
PELIC3 <- left_join(PELIC2, PELIC_scr1, by = "anon_id")

PELIC_clean <- PELIC3 %>% 
  filter(native_language == "Korean", text_len > 43, 475 > text_len) %>% 
  relocate(c(text, tokens),.after = Writing_Sample)

PELIC_clean <- PELIC_clean %>% 
  mutate(rownum = c(1:3549)) %>% 
  relocate(rownum, .after = anon_id)



```

#### 1. Lexical Diversity


```{r}

# cleaning POS data

lemPOS1 <- PELIC_clean %>% 
  mutate(tok_lem_POS = tok_lem_POS %>% 
           str_remove_all("\\[\\(") %>% 
           str_remove_all("\\)\\]") %>% 
           str_remove_all("\\(") %>% 
           str_remove_all("'") %>% 
           str_split("\\),")) %>% 
  unnest(tok_lem_POS) %>% 
  separate(tok_lem_POS, into = c('token', 'lemma', 'POS'), sep = ',') %>% 
  select(-token) %>% 
  filter(!(lemma == ""))

mean_level <- PELIC_clean %>% 
  group_by(anon_id) %>% 
  summarize(mean_lev = round(mean(level_id)))

### FINAL!!!

PELIC_TTR <- lemPOS1 %>% 
  select(-c(course_id, question_id), -c(native_language:tokens)) %>%
  group_by(rownum) %>% 
  unite(lemPOS, lemma, POS, sep = ",") %>% 
  group_by(rownum, level_id) %>% 
  summarize(lemPOS_type = length(unique(lemPOS)),
            lemPOS_token = length(lemPOS)) %>%
  group_by(level_id) %>% 
  summarize(mean_text_len = mean(lemPOS_token),
    TTR = round(mean(lemPOS_type/lemPOS_token), 4))


```


#### 2. Syntactic Complexity


```{r}

PELIC_syncom <- lemPOS1 %>% 
  group_by(anon_id) %>% 
  mutate(sent_num = cumsum(str_detect(lemma, regex("\\."))) %>% add(1)) %>% 
  ungroup() %>% 
  select(-c(rownum:course_id), -question_id, -c(native_language:tokens)) %>% 
  unite(lemPOS, lemma, POS, sep = ",") %>% 
  group_by(anon_id, sent_num) %>% 
  mutate(sent_len = length(anon_id),
         mean_texlen = mean(text_len)) %>% 
  group_by(level_id) %>% 
  summarize(mean_sent_len = mean(sent_len))

```


## PELIC final

```{r}
PELIC_TTR
PELIC_syncom
```



## Comparison between KLC and PELIC

```{r}
KLC_TTR
PELIC_TTR %>% 
  mutate(level_id = as.character(level_id)) %>% 
  rename(Level = level_id)

KLC_syncom %>% 
  mutate_all(funs(str_replace(., "A2", "2"))) %>% 
  mutate_all(funs(str_replace(., "B1", "3"))) %>% 
  mutate_all(funs(str_replace(., "B2", "4"))) %>% 
  mutate_all(funs(str_replace(., "C1", "5"))) %>% 
  mutate(mean_sent_len = as.numeric(mean_sent_len)) %>% 
  mutate(mean_sent_len = round(mean_sent_len, 4))

PELIC_syncom %>% 
  mutate(level_id = as.character(level_id)) %>% 
  rename(Level = level_id) %>% 
  mutate(mean_sent_len = round(mean_sent_len, 4))
```




### Data Visualization


#### Lexical diversity

```{r}
KLC_TTR %>% 
  ggplot(aes(x = Level, y = TTR * 1000, group = 1)) + 
  geom_point(color = "salmon") +
  geom_line(color = "salmon") +
  geom_bar(aes(y = mean_text_len), fill = "dark green", stat = 'identity') +
  scale_y_continuous(name = "mean_text_len", sec.axis = sec_axis(~. *0.001, name = "TTR")) +
  labs(title = "Korean Learner Corpus TTR & Mean Length of Essay")

PELIC_TTR %>% 
  ggplot(aes(x = level_id, y = TTR*500, group = 1)) + 
  geom_point(color = "salmon") +
  geom_line(color = "salmon") +
  geom_bar(aes(y = mean_text_len), fill = "dark green", stat = 'identity') +
  scale_y_continuous(name = "mean_text_len", sec.axis = sec_axis(~. *0.002, name = "TTR")) +
  labs(title = "PELIC TTR & Mean Length of Essay")

```


#### Syntactic complexity


```{r}

KLC_syncom %>% 
  ggplot(aes(x = Level, y = mean_sent_len, group = 1)) + 
  geom_point(color = "red") + 
  geom_path(color = "red") + 
  scale_y_continuous(limits = c(5,33))

PELIC_syncom %>% 
  ggplot(aes(x = level_id, y = mean_sent_len, group = 1)) + 
  geom_point(color = "red") + 
  geom_path(color = "red") + 
  scale_y_continuous(limits = c(5,33))

```