Dedup (aka DISTINCT) support on PhysicalPlan

Add dedup

crates/sql-parser/src/ast/sql.rs

@@ -48,12 +48,20 @@ impl SqlAst {
     }
 }
 
+/// A DISTINCT clause in a SQL SELECT statement
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum SqlDistinct {
+    No,
+    Yes,
+}
+
 /// A SELECT statement in the SQL query language
 #[derive(Debug)]
 pub struct SqlSelect {
     pub project: Project,
     pub from: SqlFrom,
     pub filter: Option<SqlExpr>,
+    pub distinct: SqlDistinct,
 }
 
 impl SqlSelect {
@@ -63,6 +71,7 @@ impl SqlSelect {
                 project: self.project.qualify_vars(alias.clone()),
                 filter: self.filter.map(|expr| expr.qualify_vars(alias.clone())),
                 from: self.from,
+                distinct: self.distinct,
             },
             SqlFrom::Join(..) => self,
         }

crates/sql-parser/src/parser/sql.rs

@@ -127,6 +127,16 @@
 //!     ;
 //! ```
 
+use super::{
+    errors::SqlUnsupported, parse_expr_opt, parse_ident, parse_literal, parse_parts, parse_projection, RelParser,
+    SqlParseResult,
+};
+use crate::ast::sql::SqlDistinct;
+use crate::ast::{
+    sql::{SqlAst, SqlDelete, SqlInsert, SqlSelect, SqlSet, SqlShow, SqlUpdate, SqlValues},
+    SqlIdent,
+};
+use sqlparser::ast::Distinct;
 use sqlparser::{
     ast::{
         Assignment, Expr, GroupByExpr, ObjectName, Query, Select, SetExpr, Statement, TableFactor, TableWithJoins,
@@ -136,16 +146,6 @@ use sqlparser::{
     parser::Parser,
 };
 
-use crate::ast::{
-    sql::{SqlAst, SqlDelete, SqlInsert, SqlSelect, SqlSet, SqlShow, SqlUpdate, SqlValues},
-    SqlIdent,
-};
-
-use super::{
-    errors::SqlUnsupported, parse_expr_opt, parse_ident, parse_literal, parse_parts, parse_projection, RelParser,
-    SqlParseResult,
-};
-
 /// Parse a SQL string
 pub fn parse_sql(sql: &str) -> SqlParseResult<SqlAst> {
     let mut stmts = Parser::parse_sql(&PostgreSqlDialect {}, sql)?;
@@ -362,7 +362,7 @@ fn parse_set_op(expr: SetExpr) -> SqlParseResult<SqlSelect> {
 fn parse_select(select: Select) -> SqlParseResult<SqlSelect> {
     match select {
         Select {
-            distinct: None,
+            distinct,
             top: None,
             projection,
             into: None,
@@ -381,12 +381,18 @@ fn parse_select(select: Select) -> SqlParseResult<SqlSelect> {
             && cluster_by.is_empty()
             && distribute_by.is_empty()
             && sort_by.is_empty()
-            && named_window.is_empty() =>
+            && named_window.is_empty()
+            && matches!(distinct, None | Some(Distinct::Distinct)) =>
         {
             Ok(SqlSelect {
                 project: parse_projection(projection)?,
                 from: SqlParser::parse_from(from)?,
                 filter: parse_expr_opt(selection)?,
+                distinct: if distinct.is_some() {
+                    SqlDistinct::Yes
+                } else {
+                    SqlDistinct::No
+                },
             })
         }
         _ => Err(SqlUnsupported::feature(select).into()),