## ----include = FALSE----------------------------------------------------------
knitr::opts_chunk$set(
  collapse = TRUE,
  message = FALSE,
  warning = FALSE,
  comment = "#>"
)

## ----setup--------------------------------------------------------------------
library(tidyEmoji)
library(dplyr)

## -----------------------------------------------------------------------------
awkward <- tibble::tibble(
  case = c("plain", "skin tone", "flag", "ZWJ family", "keycap",
           "bare heart", "no emoji"),
  text = c(
    "ship it \U0001f600 today",
    "nice work \U0001f44d\U0001f3fd",
    "landed in \U0001f1fa\U0001f1f8 already",
    paste("the whole",
          intToUtf8(c(0x1F468, 0x200D, 0x1F469, 0x200D,
                      0x1F467, 0x200D, 0x1F466)),
          "came"),
    paste("step", intToUtf8(c(0x31, 0xFE0F, 0x20E3)), "first"),
    paste("love it", intToUtf8(0x2764)),
    "nothing to see here"
  )
)

awkward %>%
  emoji_token_cost(text) %>%
  select(case, .emoji_n, .emoji_codepoints, .emoji_bytes,
         .emoji_token_estimate)

## -----------------------------------------------------------------------------
one <- awkward %>% filter(case == "ZWJ family")

for (p in c("keep", "shortcode", "name", "placeholder", "strip")) {
  cat(format(p, width = 12), "|",
      emoji_sanitize(one, text, policy = p)$text, "\n")
}

## -----------------------------------------------------------------------------
restores <- function(policy, data = one) {
  sent <- emoji_sanitize(data, text, policy = policy)
  back <- text_to_emoji(sent, text)$text
  identical(back, data$text)
}

vapply(c("keep", "shortcode", "name", "placeholder", "strip"),
       restores, logical(1))

## -----------------------------------------------------------------------------
awkward %>%
  mutate(
    sanitized = emoji_sanitize(., text, policy = "shortcode")$text,
    restored  = text_to_emoji(tibble::tibble(text = sanitized), text)$text,
    exact     = restored == text
  ) %>%
  select(case, sanitized, exact)

## -----------------------------------------------------------------------------
hearts <- tibble::tibble(
  spelling = c("bare U+2764", "qualified U+2764 U+FE0F"),
  text = c(paste("love it", intToUtf8(0x2764)),
           paste("love it", intToUtf8(c(0x2764, 0xFE0F))))
)

hearts %>% emoji_summary(text)

## -----------------------------------------------------------------------------
sanitized_hearts <- emoji_sanitize(hearts, text, policy = "shortcode")
sanitized_hearts$text
text_to_emoji(sanitized_hearts, text)$text == hearts$text

## -----------------------------------------------------------------------------
show_wrap <- function(w) {
  sent <- emoji_sanitize(one, text, policy = "shortcode", wrap = w)
  back <- text_to_emoji(sent, text)$text
  cat(format(w, width = 9), "|",
      if (identical(back, one$text)) "restored    " else "NOT restored",
      "|", back, "\n")
}

invisible(lapply(c(":{x}:", "[:{x}:]", "<{x}>", "::{x}::"), show_wrap))

## -----------------------------------------------------------------------------
corpus <- tibble::as_tibble(
  utils::read.csv(
    system.file("extdata", "ata_tweets.csv", package = "tidyEmoji"),
    encoding = "UTF-8", stringsAsFactors = FALSE
  )
)

prepared <- corpus %>%
  emoji_sentiment(full_text) %>%          # how positive were the emoji
  emoji_risk(full_text) %>%               # how much annotators disagreed
  emoji_token_cost(full_text) %>%         # what they cost in the prompt
  mutate(prompt_text = emoji_sanitize(., full_text,
                                      policy = "shortcode")$full_text)

prepared %>%
  filter(.emoji_n > 0) %>%
  select(.emoji_n, .emoji_sentiment, .emoji_ambiguity_mean,
         .emoji_token_estimate, prompt_text) %>%
  head(3)

## -----------------------------------------------------------------------------
before <- corpus %>% emoji_token_cost(full_text)
after  <- prepared %>%
  select(full_text = prompt_text) %>%
  emoji_token_cost(full_text)

tibble::tibble(
  stage = c("original", "shortcode"),
  emoji = c(sum(before$.emoji_n), sum(after$.emoji_n)),
  emoji_bytes = c(sum(before$.emoji_bytes), sum(after$.emoji_bytes)),
  est_tokens = c(sum(before$.emoji_token_estimate),
                 sum(after$.emoji_token_estimate))
)

## -----------------------------------------------------------------------------
emoji_provenance() %>% glimpse()